diff --git a/.argent/flows/restore-pipeline-speedup-benchmark.yaml b/.argent/flows/restore-pipeline-speedup-benchmark.yaml new file mode 100644 index 0000000000..477e6ecffd --- /dev/null +++ b/.argent/flows/restore-pipeline-speedup-benchmark.yaml @@ -0,0 +1,154 @@ +steps: + - echo: Restart the disposable API 37 emulator app and restore a selected + benchmark archive. + - launch: com.rajarsheechatterjee.LNReader.debug + - tool: await-ui-element + args: + condition: visible + selector: + text: DEVELOPMENT SERVERS + - echo: Select the configured local Metro server and wait for the reader screen. + - tap: + x: 0.5 + y: 0.744 + - echo: Dismiss the development menu to reach the reader home screen. + - tap: + text: Continue + - echo: Close the developer overlay and complete the fresh-install welcome setup. + - tap: + text: Close + - tool: await-ui-element + args: + condition: visible + selector: + text: Welcome + - echo: Keep the default theme and finish onboarding to open the reader. + - tap: + id: button-text + - tool: await-ui-element + args: + condition: visible + selector: + text: New update available + - echo: Defer the optional app update and continue with the current debug build. + - tap: + id: button-text + - tool: await-ui-element + args: + condition: visible + selector: + text: Search library + - echo: Open More to reach the local backup controls. + - tap: + x: 0.9 + y: 0.929 + - tool: await-ui-element + args: + condition: visible + selector: + text: Task Queue + - echo: Open Settings and then the Backup screen. + - tap: + text: Settings + - tool: await-ui-element + args: + condition: visible + selector: + text: Settings + - echo: Open the Backup settings screen. + - tap: + text: Backup + - tool: await-ui-element + args: + condition: visible + selector: + text: Local Backup + - echo: Choose the local restore action and select the v2 benchmark archive in + Android's document picker. + - tap: + text: Restore backup + - tool: await-ui-element + args: + condition: visible + selector: + text: Recent files + - echo: Open storage locations and navigate to Downloads. + - tap: + text: Show roots + - tap: + x: 0.335 + y: 0.211 + - tool: await-ui-element + args: + condition: visible + selector: + text: lnreader_backup.zip + - echo: Select the validated v2 test archive. + - tap: + id: android:id/title + - tool: await-ui-element + args: + condition: visible + selector: + text: Changes already restored + - echo: Confirm the restore and wait for the local restore task to complete. + - tap: + x: 0.795 + y: 0.578 + - echo: Decline notifications because the restore benchmark does not need them. + - tap: + id: com.android.permissioncontroller:id/permission_deny_button + - echo: Open Task Queue and verify the restore result and progress. + - tap: + role: StaticText + - tool: await-ui-element + args: + condition: visible + selector: + text: Settings + - echo: Return to More and open the queue. + - tap: + role: StaticText + - tool: await-ui-element + args: + condition: visible + selector: + text: Task Queue + - tap: + text: Task Queue + - tool: await-ui-element + args: + condition: visible + selector: + text: Local Restore + + - tool: await-ui-element + args: + condition: visible + selector: + text: No running tasks + - tool: button + args: + button: back + - tap: + x: 0.1 + y: 0.9293 + - tap: + text: Search library + - tool: keyboard + args: + text: Shadow Slave + - tool: button + args: + button: back + - tap: + text: Currently Reading + - tool: gesture-swipe + args: + fromX: 0.9 + fromY: 0.167 + toX: 0.15 + toY: 0.167 + durationMs: 300 + - tap: + text: Done Downloading diff --git a/.argent/flows/restore-v3-benchmark.yaml b/.argent/flows/restore-v3-benchmark.yaml new file mode 100644 index 0000000000..75f0d0b055 --- /dev/null +++ b/.argent/flows/restore-v3-benchmark.yaml @@ -0,0 +1,95 @@ +executionPrerequisite: LNReader's Android development build is on the DEVELOPMENT SERVERS screen, Metro is running at http://192.168.0.214:8081, and the matched v3 archive is in Downloads as lnreader_backup_v3.zip. + +steps: + - echo: Connect the fresh development client to the local Metro server. + - tap: + text: http:// + - tool: keyboard + args: + text: http://192.168.0.214:8081 + - tap: + text: Connect + - await: { visible: { text: Continue } } + - await: { idle: true } + - tap: + text: Continue + - await: { visible: { text: Close } } + - await: { idle: true } + - tap: + text: Close + - await: { visible: { text: Welcome } } + - await: { idle: true } + - echo: Keep the default theme and complete onboarding. + - tap: + text: Complete + - await: { visible: { text: New update available } } + - await: { idle: true } + - echo: Defer the optional update and use the current debug build. + - tap: + text: Later + - await: { visible: { text: Search library } } + - await: { idle: true } + - echo: Open More, then Settings and Backup. + - tap: + text: More + - await: { visible: { text: Settings } } + - await: { idle: true } + - tap: + text: Settings + - await: { visible: { id: appbar-content-title-text, text: Settings } } + - await: { idle: true } + - echo: Open the Backup settings screen. + - tap: + text: Backup + - await: { visible: { text: Local Backup } } + - await: { idle: true } + - tap: + text: Restore backup + - await: { visible: { text: lnreader_backup_v3.zip } } + - await: { idle: true } + - echo: Select the matched v3 archive; this fixture has downloaded-file restore disabled. + - tap: + id: android:id/title + - await: { visible: { text: Changes already restored } } + - await: { idle: true } + - echo: Confirm the restore and inspect its queued progress. + - tap: + text: Restore + - await: + visible: + id: 'com.android.permissioncontroller:id/permission_deny_button' + timeout: 15000 + - await: { idle: true } + - echo: Decline notifications because this data restore does not use them. + - tap: + id: com.android.permissioncontroller:id/permission_deny_button + - await: { visible: { text: Local Backup } } + - await: { idle: true } + - echo: Return through Settings to the task queue while the restore runs. + - tool: button + args: + button: back + - await: { visible: { id: appbar-content-title-text, text: Settings } } + - await: { idle: true } + - echo: Return to More to open Task Queue. + - tool: button + args: + button: back + - await: { visible: { text: Task Queue } } + - await: { idle: true } + - echo: Open Task Queue and capture the running restore state. + - tap: + text: Task Queue + - await: { visible: { text: Local Restore } } + - await: { visible: { text: No running tasks }, timeout: 300000 } + - await: { idle: true } + - echo: Return to Library after the local restore task finishes. + - tool: button + args: + button: back + - await: { visible: { text: Task Queue } } + - await: { idle: true } + - tap: + text: Library + - await: { visible: { text: Search library } } + - await: { idle: true } diff --git a/benchmark-comparison.md b/benchmark-comparison.md new file mode 100644 index 0000000000..f75c338c0e --- /dev/null +++ b/benchmark-comparison.md @@ -0,0 +1,31 @@ +## Backup restore benchmark: v2 vs v3 + +Compared the legacy v2 backup (`1.bench.zip`) with the new v3 backup (`2.bench.zip`). Both runs restored **7,323 novels** with **0 failures**, 9 categories, and 14 plugins. + +### End-to-end result + +| Metric | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Total restore time | 5m 15.6s | 4m 11.0s | **64.7s faster (20.5%)** | + +Total time is measured from `local:start` through `local:finalize:done`. + +### Restore phases + +| Phase | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Copy | 12.8s | 13.2s | 0.5s slower | +| Outer unzip | 46.3s | 30.2s | **16.1s faster (34.7%)** | +| Novel validation | 46.7s | 24.9s | **21.9s faster (46.8%)** | +| Novel restore loop | 206.8s | 179.0s | **27.8s faster (13.4%)** | + +### Novel restore timing breakdown + +| Operation | Legacy v2 | New v3 | Change | +|---|---:|---:|---:| +| Read | 52.5s | 16.6s | **35.9s faster (68.4%)** | +| Parse | 45.5s | 32.7s | **12.8s faster (28.2%)** | +| Database | 135.5s | 136.7s | 1.3s slower (0.9%) | +| Covers | 19.6s | 17.8s | 1.8s faster (9.0%) | + +The v3 run reduced end-to-end restore time by **about one minute**. The largest per-operation improvement was reading novel data, while database time was effectively unchanged. The phase durations and per-operation timings are reported separately from the benchmark logs; the operation timings should not be summed as if they were sequential phases. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..c05f9b78f3 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,48 @@ +# Backup and restore benchmarks + +`backup-restore-full.txt` is the running record for full-library restore runs. `backup-restore-small.txt` holds reduced-archive runs. When the user supplies new benchmark output, add its results to the appropriate file in the same task. Treat the user’s log as the source of truth. Preserve the run label, backup format, code-path description, archive metadata, and any validation or image-optimization variant they identify. Do not invent missing details. + +## Entry format + +Append a dated section to the relevant file. Name the run by code path, backup format, and variant, for example `integrated restore-perf + Optimize Import v3, optimized images`. + +In `backup-restore-full.txt`, keep one consolidated table before the per-run detail entries. Add one row for each run discussed, including new user-provided runs. Label backup format, image variant, and code path separately; use `—` for missing values. Retain older entries rather than replacing them. + +For each per-run entry, include: + +- Input: backup format, novel count, chapter count, and selected sections when known. +- Archive: entry count, compressed bytes, and uncompressed bytes when logged. +- Total restore time, plus novel-pipeline and `restoreData` times when the events are present. +- Copy, outer unzip, novel JSON read, parse, database, cover, category, settings, plugin, selected-file, and finalization timings when available. +- Restored and failed novel/category counts, plugin count, and cover candidate/found/copied/missing/failure counts when available. +- Database counters and validation/image variants when present. +- A short comparison only where the runs are meaningfully matched. State which archive metadata matches. Label comparisons across different backup contents or code paths as run-level observations, not causal results. +- When deriving a stage span from event timestamps rather than a logged `duration:` interval, label it as calculated and keep the raw event intervals alongside it. + +Keep raw measurements in milliseconds or seconds as emitted, with readable equivalents where helpful. Use decimal seconds to two places in summaries. Keep byte counts as bytes; optional MB figures must say whether they are decimal MB or MiB. Do not round away differences that matter to a comparison. + +## Log labels and shorthand + +- `v2` / `v3`: backup manifest format versions, not app or code versions. +- `local:start` to `local:finalize:done`: end-to-end local restore duration reported by `Total time`. +- `local:copy:done`: source archive copy into app cache has completed. +- `local:outer-unzip:done`: the outer backup archive has been extracted. The JSON payload gives entry counts and compressed/uncompressed sizes when that instrumentation is available. +- `restoreData`: manifest, library, categories, settings, and plugin-registry restore work. +- `restoreData:novels:pipeline:done`: novel-file processing pipeline completion. `total` counts processed novel JSON files, not restored novel rows. +- `readMs` / `parseMs` / `databaseMs` / `coverProcessingMs`: accumulated time for novel-file reads, JSON parsing, database restore batches, and cover processing. Cover time can include concurrent native copies; it is elapsed stage time, not a sum of per-file CPU time. +- `databaseBatches` and `batchCount`: number of non-empty database restore batches in the emitting implementation. +- `uniqueInputChapterCount`: Optimize Import deduplicates chapter identities before counting. Earlier restore-perf logs accumulated raw chapter rows despite using this same field name. Preserve the original key and explain its implementation-specific meaning. +- `chapterWriteChunkCalls` / `chapterWriteRowsAttempted`: database insert statements and input rows submitted to those statements. Different implementations may batch these differently, so compare elapsed database time and workload before comparing counts. +- `restoreFailures`: a bounded sample of per-novel database failures, not necessarily a complete count of all errors. +- `restoreData:novels:validation:start`: a timing event only. Its `duration:` value measures the gap since the previous log event, not validation runtime; there is no matching validation-complete event in these logs. +- `local:selected-archive-unzip:done`: extraction of one selected inner archive. `local:selected-archives:done` marks completion of those extractions. +- `local:downloaded-files:done`: downloaded novel files have been restored. `local:selected-files:done` marks completion of selected-file handling. +- Durations are wall-clock elapsed times from `performance.now()`. The `duration:` value is the interval since the preceding benchmark event. Adjacent stage intervals and end-to-end totals can differ slightly due to logging and omitted setup or completion work. + +## Comparisons and data quality + +Prefer runs with matching reported archive metadata, device, reset state, and workload. Matching file counts and byte counts mean the reported metadata matches; it does not prove the archives are byte-for-byte identical. Backup format, image optimization, validation settings, app code path, and run order can all affect timing. Record those distinctions explicitly. + +When showing percentage deltas, name the exact baseline in the table caption and calculate `(run value / baseline value - 1) × 100`. Positive values mean slower than baseline; negative values mean faster. If multiple Restore-perf runs exist, identify the specific one rather than calling it simply “Restore-perf”. + +Preserve each run separately. Do not replace older values with a new result or combine runs into an average unless the user asks for that analysis. If a log omits an archive size, code revision, device, or setting, write `not provided` rather than inferring it. When the user supplies more benchmark information, append it here and update this guide only if the format or shorthand changes. diff --git a/benchmarks/backup-restore-full.txt b/benchmarks/backup-restore-full.txt new file mode 100644 index 0000000000..7461cc37f4 --- /dev/null +++ b/benchmarks/backup-restore-full.txt @@ -0,0 +1,542 @@ +LNReader full backup restore benchmark +======================================= + +Input archive +------------- +- Path: omitted +- Size: 1,495,825,147 bytes +- Archive entries: 7,329 +- Novel JSON files: 7,323 +- Chapters: 2,699,763 +- Version format: 2 +- Sections: library, settings, plugins, downloadedFiles +- Archive validation: unzip -tq passed + +Android restore +---------------- +- Device: Android emulator, API 37 +- App data reset immediately before the run with pm clear +- Code path: current working tree, including the restore batching fix +- Task state: succeeded +- Task completion text: Restored 7323 novels with warnings: Missing plugins: archiveofourown, boxnovel, pawread, zetroTL, ranobes, foxaholic +- Task database start: 1790002925493 ms +- Task database completion: 1790003663424 ms +- Restore elapsed: 737.931 s (12 min 17.931 s) + +Database verification +--------------------- +- Novel rows: 7,323 +- Chapter rows: 2,699,763 +- Novels in library: 5,987 +- Task Queue completed with no remaining active task + +Host SQL reference +------------------ +These measurements came from the earlier full archive harness, which used the same +novel and chapter counts but had downloadedFiles=false. They are reference SQL timings, +not Android timings for this fresh archive. + +- Triggers on: 22.370037 s wall time +- Triggers off: 15.962422 s wall time +- Trigger delta: +6.407615 s wall time + +Section performance.now() timings +--------------------------------- +These values came from the earlier instrumented run. The app now emits the same +event names and measurements from the Metro client log for new benchmark runs. +- Input: lnreader_backup.zip, 1.50 GB, 7,323 novels, 2,699,763 chapters, 7,171 covers +- Copy into app cache: 12,034.094 ms +- Outer archive unzip: 32,433.460 ms +- Manifest load: 215.892 ms +- Novel restore section: 670,926.107 ms + - Novel JSON read and parse: 87,780.6 ms + - Novel database restore: 541,124.3 ms + - Cover restore: 39,542.1 ms +- Categories: 1,545.366 ms +- Settings: 195.745 ms +- Plugins: 345.972 ms +- Selected-file restore: 215.549 ms +- Plugin finalization: 88.977 ms +- Instrumented local:start to local:finalize:done: 718,001.758 ms + (11 min 58.002 seconds) + +Notes +----- +The first device attempt was cancelled after the restore appeared stalled at the +25-novel batch. The restore loop had an inverted directory-existence branch, so it +skipped the real archive and reported zero novels. After correcting that branch and +using single-novel batches when chapter ID mappings are required, this full run +completed and restored all rows. + +2026-09-21 standard backup +-------------------------- +- Input: standard backup, 7,323 novels +- local:start to local:copy:done: 9,597.023 ms +- local:copy:done to local:outer-unzip:done: 35,899.458 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 242.889 ms +- restoreData:manifest:loaded to restoreData:novels:done: 528,709.872 ms + - Novel JSON read and parse: 525,590.4 ms + - Novel database restore: 350,619.6 ms + - Cover restore: 29,387.5 ms +- restoreData:novels:done to restoreData:categories:start: 0.203 ms +- Categories: 1,449.609 ms +- Settings: 196.915 ms +- Plugins: 542.391 ms +- restoreData:done to local:restore-data:done: 0.088 ms +- local:restore-data:done to local:selected-files:done: 228.394 ms +- local:selected-files:done to local:finalize:done: 250.896 ms +- Instrumented local:start to local:finalize:done: 577,119.467 ms + (9 min 37.119 seconds) + +2026-09-21 new backup +--------------------- +- Input: new backup, 7,323 novels +- local:start to local:copy:done: 4,413.636 ms +- local:copy:done to local:outer-unzip:done: 24,305.590 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 218.351 ms +- restoreData:manifest:loaded to restoreData:novels:done: 640,206.997 ms + - Novel JSON read and parse: 638,024.1 ms + - Novel database restore: 481,033.4 ms + - Cover restore: 239.2 ms +- restoreData:novels:done to restoreData:categories:start: 0.135 ms +- Categories: 1,572.266 ms +- Settings: 218.562 ms +- Plugins: 581.890 ms +- restoreData:done to local:restore-data:done: 0.089 ms +- local:restore-data:done to local:selected-files:done: 39,610.108 ms +- local:selected-files:done to local:finalize:done: 939.012 ms +- Instrumented local:start to local:finalize:done: 712,068.552 ms + (11 min 52.069 seconds) + +2026-09-21 user-provided v3 restore +------------------------------------ +- Input: newly created backup, 7,323 novels +- local:start to local:copy:done: 11,978.127 ms +- local:copy:done to local:outer-unzip:done: 32,203.571 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 222.876 ms +- restoreData:manifest:loaded to restoreData:novels:done: 500,737.193 ms + - Novel JSON read: 145,315.2 ms + - Novel JSON parse: 4,759.4 ms + - Novel database restore: 336,082.9 ms + - Cover restore: 12,741.0 ms +- restoreData:novels:done to restoreData:categories:start: 0.184 ms +- Categories: 1,472.507 ms +- Settings: 202.641 ms +- Plugins: 398.214 ms +- restoreData:done to local:restore-data:done: 0.087 ms +- local:restore-data:done to local:selected-archives:done: 187.524 ms +- local:selected-archives:done to local:downloaded-files:done: 0.135 ms +- local:downloaded-files:done to local:selected-files:done: 0.068 ms +- local:selected-files:done to local:finalize:done: 277.116 ms +- Instrumented local:start to local:finalize:done: 547,680.386 ms + (9 min 7.680 seconds) + +Notes +----- +- This run's log did not include chapter-row verification or archive size. +- The database stage was 30.133% below the earlier 481,033.4 ms downloaded-file baseline. + +2026-09-21 v3 restore with novel batch size 100 +------------------------------------------------ +- Input: newly created backup, 7,323 novels +- Restore novel batch size: 100 +- local:start to local:copy:done: 15,507.341 ms +- local:copy:done to local:outer-unzip:done: 31,761.735 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 242.719 ms +- restoreData:manifest:loaded to restoreData:novels:done: 534,061.144 ms + - Novel JSON read: 153,516.0 ms + - Novel JSON parse: 4,989.5 ms + - Novel database restore: 349,994.5 ms + - Cover restore: 23,412.8 ms +- restoreData:novels:done to restoreData:categories:start: 21.725 ms +- Categories: 1,488.275 ms +- Settings: 200.373 ms +- Plugins: 518.336 ms +- restoreData:done to local:restore-data:done: 0.085 ms +- local:restore-data:done to local:selected-archives:done: 215.547 ms +- local:selected-archives:done to local:downloaded-files:done: 0.143 ms +- local:downloaded-files:done to local:selected-files:done: 0.102 ms +- local:selected-files:done to local:finalize:done: 1,194.132 ms +- Instrumented local:start to local:finalize:done: 585,211.968 ms + (9 min 45.212 seconds) + +- Compared with the preceding single run, batch size 100 was 6.853% slower overall. +- Its database stage was 4.139% slower and its cover stage was 83.760% slower. + +2026-09-21 latest restore run +---------------------------- +- Input: archive containing 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 14,512.981 ms +- local:copy:done to local:outer-unzip:done: 20,019.571 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 240.700 ms +- restoreData:manifest:loaded to restoreData:novels:done: 365,641.159 ms + - Novel JSON read: 7,418.4 ms + - Novel JSON parse: 10,592.1 ms + - Novel database restore: 336,557.5 ms + - Cover restore: 11,020.7 ms +- restoreData:novels:done to restoreData:categories:start: 0.181 ms +- Categories: 1,560.955 ms +- Settings: 212.587 ms +- Plugins: 586.187 ms +- restoreData:done to local:restore-data:done: 0.087 ms +- local:restore-data:done to local:selected-archives:done: 26.347 ms +- local:selected-archives:done to local:downloaded-files:done: 0.101 ms +- local:downloaded-files:done to local:selected-files:done: 0.071 ms +- local:selected-files:done to local:finalize:done: 242.077 ms +- Instrumented local:start to local:finalize:done: 403,043.534 ms + (6 min 43.044 seconds) + +- Compared with the preceding v3 batch-size-100 run, this run was 31.129% faster overall. +- Its novel restore section was 31.536% faster. + +2026-09-21 follow-up restore run +-------------------------------- +- Input: archive containing 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 13,313.002 ms +- local:copy:done to local:outer-unzip:done: 19,241.735 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 245.710 ms +- restoreData:manifest:loaded to restoreData:novels:done: 346,899.858 ms + - Novel JSON read: 6,956.2 ms + - Novel JSON parse: 9,919.4 ms + - Novel database restore: 320,968.2 ms + - Cover restore: 9,007.0 ms +- restoreData:novels:done to restoreData:categories:start: 0.145 ms +- Categories: 1,368.468 ms +- Settings: 191.487 ms +- Plugins: 536.287 ms +- restoreData:done to local:restore-data:done: 0.096 ms +- local:restore-data:done to local:selected-archives:done: 23.523 ms +- local:selected-archives:done to local:downloaded-files:done: 0.095 ms +- local:downloaded-files:done to local:selected-files:done: 0.066 ms +- local:selected-files:done to local:finalize:done: 446.411 ms +- Instrumented local:start to local:finalize:done: 382,267.424 ms + (6 min 22.267 seconds) + +- Compared with the preceding latest restore run, this run was 5.155% faster overall. +- Its novel restore section was 5.126% faster. +- Experimental note: `refreshRestoredNovelStats` was disabled. + +2026-09-22 restore run +---------------------- +- Input: archive containing 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 12,275.784 ms +- local:copy:done to local:outer-unzip:done: 28,577.669 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 245.182 ms +- restoreData:manifest:loaded to restoreData:novels:done: 429,693.641 ms + - Novel JSON read: 15,954.6 ms + - Novel JSON parse: 32,186.6 ms + - Novel database restore: 368,110.1 ms + - Cover restore: 13,383.0 ms +- restoreData:novels:done to restoreData:categories:start: 0.156 ms +- Categories: 1,458.343 ms +- Settings: 255.016 ms +- Plugins: 708.748 ms +- restoreData:done to local:restore-data:done: 0.090 ms +- local:restore-data:done to local:selected-archives:done: 162.989 ms +- local:selected-archives:done to local:downloaded-files:done: 0.000 ms +- local:downloaded-files:done to local:selected-files:done: 0.063 ms +- local:selected-files:done to local:finalize:done: 434.424 ms +- Instrumented local:start to local:finalize:done: 473,810.367 ms + (7 min 53.810 seconds) + +- Compared with the preceding follow-up restore run, this run was 23.947% slower overall. +- Its novel restore section was 23.867% slower. + +2026-09-22 follow-up restore run +-------------------------------- +- Input: archive containing 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 12,739.641 ms +- local:copy:done to local:outer-unzip:done: 19,986.950 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 244.540 ms +- restoreData:manifest:loaded to restoreData:novels:done: 190,043.385 ms + - Novel JSON read: 15,484.4 ms + - Novel JSON parse: 32,363.6 ms + - Novel database restore: 131,026.5 ms + - Cover restore: 11,095.6 ms +- restoreData:novels:done to restoreData:categories:start: 0.172 ms +- Categories: 1,524.607 ms +- Settings: 202.072 ms +- Plugins: 574.076 ms +- restoreData:done to local:restore-data:done: 0.133 ms +- local:restore-data:done to local:selected-archives:done: 189.844 ms +- local:selected-archives:done to local:downloaded-files:done: 0.172 ms +- local:downloaded-files:done to local:selected-files:done: 0.099 ms +- local:selected-files:done to local:finalize:done: 339.006 ms +- Instrumented local:start to local:finalize:done: 225,846.622 ms + (3 min 45.847 seconds) + +- Compared with the preceding follow-up restore run, this run was 40.919% faster overall. +- Its novel restore section was 45.217% faster. + +2026-09-22 v2 backup restore run +--------------------------------- +- Input: v2 backup, 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 14,105.527 ms +- local:copy:done to local:outer-unzip:done: 33,360.384 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 231.974 ms +- restoreData:manifest:loaded to restoreData:novels:done: 357,730.435 ms + - Novel validation: 155,869.1 ms + - Novel restore: 201,860.0 ms + - Restore split times (74 progress intervals): min 711.0 ms, max 5,672.8 ms, average 2,727.8 ms + - Novel JSON read: 160,922.6 ms + - Novel JSON parse: 46,027.9 ms + - Novel database restore: 131,407.3 ms + - Cover restore: 17,633.4 ms +- restoreData:novels:done to restoreData:categories:start: 0.083 ms +- Categories: 1,432.408 ms +- Settings: 215.064 ms +- Plugins: 682.238 ms +- restoreData:done to local:restore-data:done: 0.113 ms +- local:restore-data:done to local:selected-archives:done: 31.628 ms +- local:selected-archives:done to local:downloaded-files:done: 0.112 ms +- local:downloaded-files:done to local:selected-files:done: 0.065 ms +- local:selected-files:done to local:finalize:done: 415.724 ms +- Instrumented local:start to local:finalize:done: 408,206.317 ms + (6 min 48.206 seconds) + +- Compared with the preceding follow-up restore run, this run was 80.745% slower overall. +- Its novel restore section was 88.236% slower. + +2026-09-22 v2 backup restore run with validation comparison +------------------------------------------------------------ +- Input: v2 backup, 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 16,434.861 ms +- local:copy:done to local:outer-unzip:done: 45,774.190 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 243.218 ms +- restoreData:manifest:loaded to restoreData:novels:done: 241,379.882 ms + - Novel validation: 43,751.8 ms + - Novel restore: 197,626.5 ms + - Restore split times (74 progress intervals): min 688.8 ms, max 12,697.9 ms, average 2,670.6 ms + - Novel JSON read: 44,899.8 ms + - Novel JSON parse: 46,070.6 ms + - Novel database restore: 133,105.8 ms + - Cover restore: 16,795.0 ms +- restoreData:novels:done to restoreData:categories:start: 0.127 ms +- Categories: 1,506.012 ms +- Settings: 260.099 ms +- Plugins: 730.673 ms +- restoreData:done to local:restore-data:done: 0.123 ms +- local:restore-data:done to local:selected-archives:done: 189.994 ms +- local:selected-archives:done to local:downloaded-files:done: 0.126 ms +- local:downloaded-files:done to local:selected-files:done: 0.078 ms +- local:selected-files:done to local:finalize:done: 423.740 ms +- Instrumented local:start to local:finalize:done: 306,943.743 ms + (5 min 6.944 seconds) + +- Compared with the 2026-09-22 v2 backup restore run, this run was 6.708% slower overall. +- Its manifest-to-novels section was 0.129% faster. + +2026-09-22 v3 backup restore run +--------------------------------- +- Input: v3 backup, 7,323 novels +- Novel restore result: count=7,323, failed=0 +- Categories: 9 restored, 0 failed +- Settings: restored +- Plugins: 14 restored +- Downloaded-file restore: completed +- local:start to local:copy:done: 13,887.011 ms +- local:copy:done to local:outer-unzip:done: 21,220.154 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 219.586 ms +- restoreData:manifest:loaded to restoreData:novels:done: 201,317.476 ms + - Novel validation: 23,868.2 ms + - Novel restore: 177,447.7 ms + - Restore split times (74 progress intervals): min 1,106.8 ms, max 4,703.1 ms, average 2,397.9 ms + - Novel JSON read: 16,470.6 ms + - Novel JSON parse: 32,987.6 ms + - Novel database restore: 135,555.4 ms + - Cover restore: 16,221.0 ms +- restoreData:novels:done to restoreData:categories:start: 0.108 ms +- Categories: 1,537.914 ms +- Settings: 195.709 ms +- Plugins: 581.576 ms +- restoreData:done to local:restore-data:done: 0.134 ms +- local:restore-data:done to local:selected-archives:done: 29.690 ms +- local:selected-archives:done to local:downloaded-files:done: 0.124 ms +- local:downloaded-files:done to local:selected-files:done: 0.115 ms +- local:selected-files:done to local:finalize:done: 366.705 ms +- Instrumented local:start to local:finalize:done: 239,356.876 ms + (3 min 59.357 seconds) + +- Compared with the preceding v2 backup restore run, this run was 22.019% faster overall. +- Its manifest-to-novels section was 16.597% faster. + +User-supplied Metro benchmark runs +================================== +These summaries preserve the user-provided runs from the comparison conversation. +All total times are `local:start` to `local:finalize:done` unless marked otherwise. + +Consolidated comparison of runs supplied in the conversation +------------------------------------------------------------ +Times are seconds except archive sizes in bytes. Percentages beside total and +novel-pipeline durations compare with the Restore-perf v2, before-integration +baseline (155.09 s total; 100.62 s novel pipeline). A positive percentage means +slower than baseline; a negative percentage means faster. The four-part novel-stage +cell is read / parse / database / covers. The final stage cell is manifest / +categories / settings / plugins / selected-archive unzip / finalization, in +milliseconds. `—` means the log did not provide the value. Matching archive +metadata is not proof that archive contents are identical. + +| Run | Backup format | Image variant | Code path | Archive compressed / uncompressed bytes | Total / novel pipeline (s, delta vs baseline) | Copy / outer unzip (s) | Read / parse / DB / covers (s) | Manifest / categories / settings / plugins / selected unzip / finalization (ms) | statsRefreshCalls | +|---|---|---|---|---:|---:|---:|---:|---:|---:| +| Optimize Import v3, before integration, standard images | v3 | Standard | Optimize Import v3, before integration | 1,487,128,327 / 1,891,673,000 | 185.75 [+19.77%] / 156.00 [+55.04%] | 6.37 / 20.55 | 9.600 / 16.387 / 137.420 / 17.955 | 232.16 / 1,449.59 / 207.27 / 383.56 / 223.22 / 321.31 | — | +| Optimize Import v3, before integration, optimized images | v3 | Optimized | Optimize Import v3, before integration | 358,723,347 / 754,040,852 | 167.06 [+7.72%] / 143.83 [+42.94%] | 2.06 / 18.45 | 8.726 / 16.026 / 134.689 / 8.553 | 231.22 / 1,448.99 / 194.07 / 382.88 / 218.02 / 249.68 | — | +| Restore-perf v3, earlier run | v3 | Not provided | Restore-perf v3 | — | 173.95 [+12.16%] / 113.62 [+12.92%] | 11.49 / 46.88 | — / — / — / — | 239.68 / 460.17 / 232.98 / 549.35 / — / 247.51 | — | +| Optimize Import v2, earlier run | v2 | Standard | Optimize Import v2 | 1,493,783,823 / 2,369,830,658 | 363.75 [+134.54%] / 310.91 [+208.99%] | 10.75 / 39.31 | 133.648 / 23.260 / 134.773 / 19.029 | 241.98 / 1,443.00 / 207.46 / 547.96 / 30.58 / 305.31 | — | +| Optimize Import v2, later run | v2 | Standard | Optimize Import v2 | 1,493,783,823 / 2,369,830,658 | 147.80 [-4.70%] / 95.90 [-4.69%] | 7.36 / 42.85 | 22.843 / 23.021 / 35.557 / 14.435 | 227.77 / 417.66 / 203.48 / 567.60 / 29.95 / 229.16 | — | +| Restore-perf v2, before integration, baseline | v2 | Standard | Restore-perf v2, before integration | 1,493,783,823 / 2,369,830,658 | 155.09 [baseline] / 100.62 [baseline] | 11.02 / 41.50 | 28.496 / 4.846 / 52.377 / 14.256 | 245.45 / 408.16 / 202.08 / 549.77 / 234.96 / 304.31 | — | +| Integrated Optimize Import v3, optimized images | v3 | Optimized | Integrated Optimize Import v3 | 358,723,347 / 754,040,852 | 88.44 [-42.98%] / 66.27 [-34.14%] | 1.66 / 18.76 | 9.028 / 16.620 / 54.760 / 10.853 | 245.18 / 422.20 / 215.95 / 414.38 / 230.08 / 210.54 | — | +| Integrated Optimize Import v3, standard images | v3 | Standard | Integrated Optimize Import v3 | 1,487,128,327 / 1,891,673,000 | 103.78 [-33.08%] / 68.77 [-31.65%] | 11.55 / 21.84 | 8.929 / 16.616 / 55.260 / 12.842 | 228.36 / 417.60 / 196.67 / 385.62 / 179.38 / 210.54 | — | +| Optimize Import v2, validation duplication | v2 | Standard | Optimize Import v2, validation duplication enabled | 1,493,783,823 / 2,369,830,658 | 140.07 [-9.68%] / 92.87 [-7.70%] | 5.62 / 39.84 | 24.999 / 15.510 / 36.369 / 15.908 | 237.98 / 304.15 / 196.92 / 652.86 / 28.75 / 290.18 | — | +| Optimize Import v3, latest run | v3 | Not provided | Optimize Import branch, v3 import | 1,487,128,327 / 1,891,673,000 | 96.45 [-37.81%] / 65.55 [-34.85%] | 8.91 / 20.32 | 13.143 / 12.800 / 49.803 / 14.627 | 217.75 / 403.71 / 205.80 / 418.95 / 175.18 / 238.62 | 74 | + +2026-10-01 Optimize Import v3, standard images, before restore-perf integration +------------------------------------------------------------------------------ +- Input: v3 backup, 7,323 novels, 2,699,763 chapters +- Archive: 7,250 entries, 1,487,128,327 compressed bytes, 1,891,673,000 uncompressed bytes +- Total 185.75 s; novel pipeline 156.00 s; restoreData span 158.05 s (timestamp-derived: manifest 236993.28 -> restoreData:done 395042.22) +- Novel timings: read 9.600 s; parse 16.387 s; database 137.420 s; covers 17.955 s +- Copy 6.37 s; outer unzip 20.55 s; manifest event 232.16 ms +- Categories 1,449.59 ms; settings 207.27 ms; plugins 383.56 ms +- Selected archive unzip 223.22 ms; finalization 321.31 ms +- 7,323 restored, 0 failed; 9 categories; 14 plugins +- Covers: 7,171 candidates, 7,171 found/copied, none missing + +2026-10-01 Optimize Import v3, optimized images, before restore-perf integration +-------------------------------------------------------------------------------- +- Input: v3 backup, 7,323 novels, 2,699,763 chapters +- Archive: 7,248 entries, 358,723,347 compressed bytes, 754,040,852 uncompressed bytes +- Total 167.06 s; novel pipeline 143.83 s; restoreData span 145.85 s (timestamp-derived: manifest 5347841.41 -> restoreData:done 5493694.08) +- Novel timings: read 8.726 s; parse 16.026 s; database 134.689 s; covers 8.553 s +- Copy 2.06 s; outer unzip 18.45 s; manifest event 231.22 ms +- Categories 1,448.99 ms; settings 194.07 ms; plugins 382.88 ms +- Selected archive unzip 218.02 ms; finalization 249.68 ms +- 7,323 restored, 0 failed; 9 categories; 14 plugins +- Covers: 7,171 candidates, 7,169 found/copied, 2 missing + +2026-10-01 Restore-perf v3, earlier run, archive variant unknown +--------------------------------------------------------------- +- Input: v3 backup, 7,323 novels; archive size and image variant not provided +- Total 173.95 s; novel pipeline 113.62 s; restoreData span 114.87 s (timestamp-derived: manifest 2132844.19 -> restoreData:done 2247710.55) +- Copy 11.49 s; outer unzip 46.88 s; manifest event 239.68 ms +- Categories 460.17 ms; settings 232.98 ms; plugins 549.35 ms +- Selected-file restore 228.30 ms; finalization 247.51 ms +- 7,323 restored, 0 failed + +2026-10-01 Optimize Import v2, earlier run +------------------------------------------ +- Input: v2 backup, 7,323 novels, 2,699,763 chapters +- Archive: 14,499 entries, 1,493,783,823 compressed bytes, 2,369,830,658 uncompressed bytes +- Total 363.75 s; novel pipeline 310.91 s +- Novel timings: read 133.648 s; parse 23.260 s; database 134.773 s; covers 19.029 s +- Copy 10.75 s; outer unzip 39.31 s; manifest event 241.98 ms +- Categories 1,443.00 ms; settings 207.46 ms; plugins 547.96 ms +- Selected archive unzip 30.58 ms; finalization 305.31 ms +- 7,323 restored, 0 failed; 9 categories; 14 plugins + +2026-10-01 Optimize Import v2, later run +---------------------------------------- +- Input: v2 backup, 7,323 novels, 2,699,763 chapters +- Archive metadata: same reported entry count and byte counts as the earlier v2 run +- Total 147.80 s; novel pipeline 95.90 s +- Novel timings: read 22.843 s; parse 23.021 s; database 35.557 s; covers 14.435 s +- Copy 7.36 s; outer unzip 42.85 s; manifest event 227.77 ms +- Categories 417.66 ms; settings 203.48 ms; plugins 567.60 ms +- Selected archive unzip 29.95 ms; finalization 229.16 ms +- 7,323 restored, 0 failed; 9 categories; 14 plugins + +2026-10-01 Restore-perf v2, before integration +---------------------------------------------- +- Input: v2 backup, 7,323 novels, 2,699,763 chapters +- Archive metadata: same reported entry count and byte counts as Optimize Import v2 +- Total 155.09 s; novel pipeline 100.62 s +- Novel timings: read 28.496 s; parse 4.846 s; database 52.377 s; covers 14.256 s +- Copy 11.02 s; outer unzip 41.50 s; manifest event 245.45 ms +- Categories 408.16 ms; settings 202.08 ms; plugins 549.77 ms +- Selected archive unzip 234.96 ms; finalization 304.31 ms +- Database: 485 batches; 7,323 upserts and identity lookups; 30,682 chapter writes; + 2,699,763 chapter rows; no restore failures +- 7,323 restored, 0 failed; 9 categories; 14 plugins + +2026-10-01 Integrated Optimize Import v3, optimized images +--------------------------------------------------------- +- Input: v3 backup, 7,323 novels, 2,699,763 chapters +- Archive: 7,248 entries, 358,723,347 compressed bytes, 754,040,852 uncompressed bytes +- Total 88.44 s; novel pipeline 66.27 s; restoreData span 67.33 s (timestamp-derived: manifest 1927462.25 -> restoreData:done 1994794.15) +- Novel timings: read 9.028 s; parse 16.620 s; database 54.760 s; covers 10.853 s +- Copy 1.66 s; outer unzip 18.76 s; manifest event 245.18 ms +- Categories 422.20 ms; settings 215.95 ms; plugins 414.38 ms +- Selected archive unzip 230.08 ms; finalization 210.54 ms +- Database: 74 upserts; 305 chapter-write chunks; 2,699,763 chapter rows; + no restore failures +- 7,323 restored, 0 failed; 9 categories; 14 plugins +- Covers: 7,171 candidates, 7,169 found/copied, 2 missing + +2026-10-01 Integrated Optimize Import v3, standard images +-------------------------------------------------------- +- Input: v3 backup, 7,323 novels, 2,699,763 chapters +- Archive: 7,250 entries, 1,487,128,327 compressed bytes, 1,891,673,000 uncompressed bytes +- Total 103.78 s; novel pipeline 68.77 s; restoreData span 69.78 s (timestamp-derived: manifest 1749439.53 -> restoreData:done 1819216.57) +- Novel timings: read 8.929 s; parse 16.616 s; database 55.260 s; covers 12.842 s +- Copy 11.55 s; outer unzip 21.84 s; manifest event 228.36 ms +- Categories 417.60 ms; settings 196.67 ms; plugins 385.62 ms +- Selected archive unzip 179.38 ms; finalization 210.54 ms +- Database: 74 upserts; 305 chapter-write chunks; 2,699,763 chapter rows; + no restore failures +- 7,323 restored, 0 failed; 9 categories; 14 plugins +- Covers: 7,171 candidates, all found/copied, none missing + +2026-10-01 Optimize Import v2, validation duplication enabled +------------------------------------------------------------ +- Input: v2 backup, 7,323 novels, 2,699,763 chapters +- Archive metadata: same reported entry count and byte counts as other Optimize Import v2 runs +- Total 140.07 s; novel pipeline 92.87 s +- Novel timings: read 24.999 s; parse 15.510 s; database 36.369 s; covers 15.908 s +- Copy 5.62 s; outer unzip 39.84 s; manifest event 237.98 ms +- Categories 304.15 ms; settings 196.92 ms; plugins 652.86 ms +- Selected archive unzip 28.75 ms; finalization 290.18 ms +- 7,323 restored, 0 failed; 9 categories; 14 plugins + +Latest user-supplied Optimize Import branch v3 import, standard image +------------------------------------------------------------------------------ +- Input: v3 backup; 7,323 restored novels, 2,699,763 unique input chapters; novel pipeline total=74 +- Archive: 7,250 entries, 1,487,128,327 compressed bytes, 1,891,673,000 uncompressed bytes; largest entry 13,982,357 bytes +- Archive metadata matches the reported standard-image v3 runs; that does not establish identical archive contents +- Total 96.45 s; novel pipeline 65.55 s; restoreData span 66.58 s (timestamp-derived: manifest 57951638.26 -> restoreData:done 58018221.85) +- Novel timings: read 13.143 s; parse 12.800 s; database 49.803 s; covers 14.627 s +- Copy 8.91 s; outer unzip 20.32 s; manifest event 217.75 ms +- Validation-start event followed the preceding log event by 3.41 ms; validation runtime was not separately measured +- Categories 403.71 ms; settings 205.80 ms; plugins 418.95 ms +- Selected archive unzip 175.18 ms (15 entries; 78,448 compressed bytes, 237,698 uncompressed bytes); finalization 238.62 ms +- Database counters: 74 novel upsert calls / 7,323 rows; 74 identity lookup calls / 7,323 rows; 305 chapter-write chunks / 2,699,763 rows; 0 chapter-mapping rows; 74 stats refresh calls for 7,323 novels; 0 novel and chapter batch fallbacks +- Database operation timings: novel upserts 2.693 s; identity lookups 2.282 s; chapter writes 37.246 s; stats refresh 5.222 s +- 7,323 restored, 0 failed; 9 categories, 0 failed; 14 plugins +- Covers: 7,171 candidates, all found/copied; none missing; 0 cover-processing failures diff --git a/benchmarks/backup-restore-small.txt b/benchmarks/backup-restore-small.txt new file mode 100644 index 0000000000..318c2fb436 --- /dev/null +++ b/benchmarks/backup-restore-small.txt @@ -0,0 +1,62 @@ +LNReader smaller backup restore benchmark +========================================== + +Input archive +------------- +- Path: omitted +- Size: 5,001,992 bytes +- Archive entries: 31 +- Novel JSON files: 25 +- Chapters: 128,003 +- Version format: 2 +- Sections: library, settings, plugins, downloadedFiles +- Archive validation: unzip -tq passed + +Android restore +---------------- +- Device: Android emulator, API 37 +- App data reset immediately before the run with pm clear +- Code path: current working tree, including the restore batching fix +- Task state: succeeded +- Task completion text: Restored 25 novels +- Task database start: 1790002706431 ms +- Task database completion: 1790002725708 ms +- Restore elapsed: 19.277 s + +Database verification +--------------------- +- Novel rows: 25 +- Chapter rows: 128,003 +- Novels in library: 20 +- Task Queue completed with no remaining active task + +Host SQL reference +------------------ +These measurements came from the earlier reduced archive harness. That archive had +downloadedFiles=false, so these are reference SQL timings, not Android timings for this +fresh archive. + +- Optimized empty-database restore: 0.552109 s +- Baseline empty-database restore: 0.471120 s +- Optimized existing-database pass: 0.678089 s +- Baseline existing-database pass: 76.428216 s + +Section performance.now() timings +--------------------------------- +These values came from the earlier instrumented run. The app now emits the same +event names and measurements from the Metro client log for new benchmark runs. +- local:start to local:copy:done: 43.696 ms +- local:copy:done to local:outer-unzip:done: 278.927 ms +- local:outer-unzip:done to restoreData:manifest:loaded: 231.428 ms +- restoreData:manifest:loaded to restoreData:novels:done: 18,221.204 ms + - Novel JSON read and parse: 987.6 ms + - Novel database restore: 16,997.0 ms + - Cover restore: 216.3 ms +- restoreData:novels:done to restoreData:categories:start: 0.130 ms +- Categories: 192.115 ms +- Settings: 214.668 ms +- Plugins: 404.450 ms +- restoreData:done to local:restore-data:done: 0.109 ms +- local:restore-data:done to local:selected-files:done: 231.884 ms +- local:selected-files:done to local:finalize:done: 140.311 ms +- Instrumented local:start to local:finalize:done: 19,959.431 ms diff --git a/drizzle/20260922120323_demonic_rattler/migration.sql b/drizzle/20260922120323_demonic_rattler/migration.sql new file mode 100644 index 0000000000..c0b084a3da --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/migration.sql @@ -0,0 +1,58 @@ +CREATE TABLE `RestoreChapterMapping` ( + `restoreRunId` text NOT NULL, + `backupNovelId` integer NOT NULL, + `backupChapterId` integer NOT NULL, + `restoredNovelId` integer NOT NULL, + `restoredChapterId` integer NOT NULL, + CONSTRAINT `fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk` FOREIGN KEY (`restoredNovelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk` FOREIGN KEY (`restoredChapterId`) REFERENCES `Chapter`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_Chapter` ( + `id` integer PRIMARY KEY AUTOINCREMENT, + `novelId` integer NOT NULL, + `path` text NOT NULL, + `name` text NOT NULL, + `releaseTime` text, + `bookmark` integer DEFAULT false, + `unread` integer DEFAULT true, + `readTime` text, + `isDownloaded` integer DEFAULT false, + `updatedTime` text, + `chapterNumber` real, + `page` text DEFAULT '1', + `position` integer DEFAULT 0, + `progress` integer, + `scanlator` text, + `timeSpent` integer DEFAULT 0, + CONSTRAINT `fk_Chapter_novelId_Novel_id_fk` FOREIGN KEY (`novelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +-- Preserve the previous AUTOINCREMENT high-water mark across the rebuild. +CREATE TEMP TABLE `__chapter_sequence` AS +SELECT COALESCE(MAX(`seq`), 0) AS `seq` +FROM `sqlite_sequence` +WHERE `name` = 'Chapter'; +--> statement-breakpoint +-- Chapters without a novel are invalid legacy data and are intentionally discarded. +INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter` WHERE EXISTS (SELECT 1 FROM `Novel` WHERE `Novel`.`id` = `Chapter`.`novelId`);--> statement-breakpoint +DROP TABLE `Chapter`;--> statement-breakpoint +ALTER TABLE `__new_Chapter` RENAME TO `Chapter`;--> statement-breakpoint +INSERT INTO `sqlite_sequence` (`name`, `seq`) +SELECT 'Chapter', `seq` FROM `__chapter_sequence` +WHERE NOT EXISTS ( + SELECT 1 FROM `sqlite_sequence` WHERE `name` = 'Chapter' +); +--> statement-breakpoint +UPDATE `sqlite_sequence` +SET `seq` = MAX(`seq`, (SELECT `seq` FROM `__chapter_sequence`)) +WHERE `name` = 'Chapter'; +--> statement-breakpoint +DROP TABLE `__chapter_sequence`; +--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `chapter_novel_path_unique` ON `Chapter` (`novelId`,`path`);--> statement-breakpoint +CREATE INDEX `chapterNovelIdIndex` ON `Chapter` (`novelId`,`position`,`page`,`id`);--> statement-breakpoint +CREATE UNIQUE INDEX `restore_chapter_mapping_unique` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`,`backupChapterId`);--> statement-breakpoint +CREATE INDEX `restore_chapter_mapping_novel_index` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`); \ No newline at end of file diff --git a/drizzle/20260922120323_demonic_rattler/snapshot.json b/drizzle/20260922120323_demonic_rattler/snapshot.json new file mode 100644 index 0000000000..fc040891a4 --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/snapshot.json @@ -0,0 +1,793 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "bcdf1b04-31ef-45ca-8a9a-8ecd417c3512", + "prevIds": [ + "2fada83f-d142-46d9-ba03-5ae1e232e3ce" + ], + "ddl": [ + { + "name": "Category", + "entityType": "tables" + }, + { + "name": "Chapter", + "entityType": "tables" + }, + { + "name": "NovelCategory", + "entityType": "tables" + }, + { + "name": "Novel", + "entityType": "tables" + }, + { + "name": "Repository", + "entityType": "tables" + }, + { + "name": "RestoreChapterMapping", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Category" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "sort", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "releaseTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "bookmark", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "unread", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "readTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isDownloaded", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "real", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "chapterNumber", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'1'", + "generated": null, + "name": "page", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "position", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "progress", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "scanlator", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "timeSpent", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "categoryId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "pluginId", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cover", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "author", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "artist", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'Unknown'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "genres", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "inLibrary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isLocal", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalPages", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersDownloaded", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersUnread", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalChapters", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastReadAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastUpdatedAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoreRunId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "novelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_Chapter_novelId_Novel_id_fk", + "entityType": "fks", + "table": "Chapter" + }, + { + "columns": [ + "restoredNovelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "restoredChapterId" + ], + "tableTo": "Chapter", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Category_pk", + "table": "Category", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Chapter_pk", + "table": "Chapter", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "NovelCategory_pk", + "table": "NovelCategory", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Novel_pk", + "table": "Novel", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Repository_pk", + "table": "Repository", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "name", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "category_name_unique", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "sort", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "category_sort_idx", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "chapter_novel_path_unique", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "position", + "isExpression": false + }, + { + "value": "page", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "chapterNovelIdIndex", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "categoryId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_category_unique", + "entityType": "indexes", + "table": "NovelCategory" + }, + { + "columns": [ + { + "value": "path", + "isExpression": false + }, + { + "value": "pluginId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_path_plugin_unique", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "pluginId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + }, + { + "value": "inLibrary", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "NovelIndex", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "url", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "repository_url_unique", + "entityType": "indexes", + "table": "Repository" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + }, + { + "value": "backupChapterId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_unique", + "entityType": "indexes", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_novel_index", + "entityType": "indexes", + "table": "RestoreChapterMapping" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/drizzle/migrations.js b/drizzle/migrations.js index beee4c381c..74735a0528 100644 --- a/drizzle/migrations.js +++ b/drizzle/migrations.js @@ -5,6 +5,7 @@ import m0001 from './20260612232322_normal_saracen/migration.sql'; import m0002 from './20260719143427_long_moondragon/migration.sql'; import m0003 from './20260727081855_calm_chimera/migration.sql'; import m0004 from './20260811071655_parched_human_torch/migration.sql'; +import m0005 from './20260922120323_demonic_rattler/migration.sql'; export default { migrations: { @@ -13,5 +14,6 @@ export default { '20260719143427_long_moondragon': m0002, '20260727081855_calm_chimera': m0003, '20260811071655_parched_human_torch': m0004, + '20260922120323_demonic_rattler': m0005, }, }; diff --git a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt index 3a957a4d08..83db1e6d94 100644 --- a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt +++ b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt @@ -28,19 +28,22 @@ import okhttp3.JavaNetCookieJar import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import java.io.BufferedInputStream +import java.io.BufferedOutputStream import java.io.File +import java.io.FileInputStream import java.io.FileOutputStream import java.io.FileWriter import java.io.IOException import java.io.InputStream import java.io.OutputStream -import java.util.UUID import java.io.PushbackInputStream +import java.util.UUID import java.util.zip.GZIPInputStream import kotlin.coroutines.coroutineContext class NativeFileModule : Module() { - private val BUFFER_SIZE = 4096 + private val BUFFER_SIZE = 64 * 1024 private val okHttpClient = OkHttpClientProvider.createClient() private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private var pendingDocumentPromise: Promise? = null @@ -108,60 +111,67 @@ class NativeFileModule : Module() { ?: throw Exception("ENOENT: could not open an input stream for '$filepath'") } - private val writeAccessByAPILevel: String - get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private fun getOutputStream(filepath: String): OutputStream { val uri = getFileUri(filepath) return reactContext?.contentResolver?.openOutputStream(uri, writeAccessByAPILevel) ?: throw Exception("ENOENT: could not open an output stream for '$filepath'") } + private val writeAccessByAPILevel: String + get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private suspend fun copyFileContent( - filepath: String, - destPath: String, - onDone: (() -> Unit)? = null, - ): Long { - try { - val inputStream = getInputStream(filepath) - var copiedBytes = 0L - try { - val outputStream = getOutputStream(destPath) - try { - val buffer = ByteArray(BUFFER_SIZE) - var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { - coroutineContext.ensureActive() - outputStream.write(buffer, 0, length) - copiedBytes += length - } - outputStream.flush() - } finally { - outputStream.close() - } - } finally { - inputStream.close() - } - if (onDone != null) { - onDone() - } - return copiedBytes - } catch (e: IOException) { - throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + + private fun localFileOrNull(filepath: String): File? { + val uri = Uri.parse(filepath) + return when { + uri.scheme == null -> File(filepath) + uri.scheme.equals(ContentResolver.SCHEME_FILE, ignoreCase = true) -> + uri.path?.let(::File) + else -> null } } - private fun contentResolver(): ContentResolver = reactContext?.contentResolver ?: throw IOException("React context is unavailable") - private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long { + + private suspend fun copyLocalFile(sourceFile: File, destinationFile: File): Long { + if (sourceFile.isDirectory) { + throw IOException("Invalid file, folder found!") + } + + val sourceSize = sourceFile.length() var copiedBytes = 0L - getInputStream(sourcePath).use { inputStream -> - outputStream.use { output -> + FileInputStream(sourceFile).channel.use { sourceChannel -> + FileOutputStream(destinationFile).channel.use { destinationChannel -> + while (copiedBytes < sourceSize) { + coroutineContext.ensureActive() + val bytesToTransfer = minOf(BUFFER_SIZE.toLong(), sourceSize - copiedBytes) + val transferredBytes = + sourceChannel.transferTo(copiedBytes, bytesToTransfer, destinationChannel) + if (transferredBytes <= 0) { + throw IOException("Could not copy the complete local file") + } + copiedBytes += transferredBytes + } + } + } + + if (destinationFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + return copiedBytes + } + + private suspend fun copyStreams( + inputStream: InputStream, + outputStream: OutputStream, + ): Long { + var copiedBytes = 0L + BufferedInputStream(inputStream, BUFFER_SIZE).use { input -> + BufferedOutputStream(outputStream, BUFFER_SIZE).use { output -> val buffer = ByteArray(BUFFER_SIZE) var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { + while (input.read(buffer).also { length = it } > 0) { coroutineContext.ensureActive() output.write(buffer, 0, length) copiedBytes += length @@ -172,6 +182,44 @@ class NativeFileModule : Module() { return copiedBytes } + private suspend fun copyFileContent( + filepath: String, + destPath: String, + onDone: (() -> Unit)? = null, + ): Long { + try { + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val copiedBytes = if (sourceFile != null && destinationFile != null) { + try { + copyLocalFile(sourceFile, destinationFile) + } catch (_: IOException) { + FileInputStream(sourceFile).use { inputStream -> + FileOutputStream(destinationFile).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + } else { + getInputStream(filepath).use { inputStream -> + getOutputStream(destPath).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + if (sourceFile != null && sourceFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + onDone?.invoke() + return copiedBytes + } catch (e: IOException) { + throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + } + } + + private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long = + copyStreams(getInputStream(sourcePath), outputStream) + private fun resolveDirectoryFile(directoryUri: String): File { val uri = Uri.parse(directoryUri) val directory = if (uri.scheme == null) { @@ -479,7 +527,28 @@ class NativeFileModule : Module() { AsyncFunction("moveFile") { filepath: String, destPath: String, promise: Promise -> coroutineScope.launch { try { - val inFile = File(filepath) + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val renamed = if ( + sourceFile != null && + destinationFile != null && + sourceFile.isFile && + !destinationFile.exists() + ) { + try { + sourceFile.renameTo(destinationFile) + } catch (_: SecurityException) { + false + } + } else { + false + } + if (renamed) { + promise.resolve(null) + return@launch + } + + val inFile = sourceFile ?: File(filepath) copyFileContent(filepath, destPath) { if (!inFile.delete()) { throw IOException("Failed to delete source file '$filepath'") diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index 75222b47a1..18ee17ae36 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -13,11 +13,24 @@ import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream class NativeZipArchiveModule : Module() { - private fun zipProcess(sourceDirPath: String, zos: ZipOutputStream) { + private fun zipProcess( + sourceDirPath: String, + zos: ZipOutputStream, + archivePrefix: String = "", + archiveEntries: MutableSet? = null, + excludedFile: File? = null, + ) { val sourceDir = File(sourceDirPath) - sourceDir.walkBottomUp().filter { it.isFile }.forEach { file -> - val zipFileName = + val excludedCanonicalPath = excludedFile?.canonicalPath + sourceDir.walkBottomUp().filter { file -> + file.isFile && file.canonicalPath != excludedCanonicalPath + }.forEach { file -> + val relativePath = file.absolutePath.removePrefix(sourceDir.absolutePath).removePrefix("/") + val zipFileName = joinArchivePath(archivePrefix, relativePath) + if (archiveEntries != null && !archiveEntries.add(zipFileName)) { + throw IllegalArgumentException("Duplicate ZIP entry: $zipFileName") + } val entry = ZipEntry("$zipFileName${(if (file.isDirectory) "/" else "")}") zos.putNextEntry(entry) file.inputStream().use { fis -> @@ -26,22 +39,95 @@ class NativeZipArchiveModule : Module() { } } + private fun joinArchivePath(prefix: String, relativePath: String): String { + val normalizedRelativePath = relativePath.replace('\\', '/').trimStart('/') + return if (prefix.isEmpty()) { + normalizedRelativePath + } else { + "$prefix/$normalizedRelativePath" + } + } + + private fun normalizeArchivePrefix(prefix: String): String { + require(prefix.isEmpty() || prefix.isNotBlank()) { "ZIP prefix must not be blank" } + val normalized = prefix.replace('\\', '/') + require(!normalized.startsWith("/") && !(normalized.length >= 2 && normalized[1] == ':')) { + "ZIP prefix must be relative" + } + val components = normalized.split('/').filter { it.isNotEmpty() && it != "." } + require(components.none { it == ".." }) { "ZIP prefix must not contain .." } + return components.joinToString("/") + } + + private fun ensureParentDirectory(file: File, createdDirectories: MutableSet) { + val parent = file.parentFile ?: return + val parentPath = parent.absolutePath + if (parentPath !in createdDirectories && (parent.exists() || parent.mkdirs())) { + createdDirectories.add(parentPath) + } + } + private fun resolveZipEntry(destination: File, entryName: String): File { + val canonicalDestination = destination.canonicalFile + val outputFile = File(destination, entryName).canonicalFile + val destinationPath = + canonicalDestination.path.let { if (it.endsWith(File.separator)) it else it + File.separator } + require(outputFile.path.startsWith(destinationPath)) { + "ZIP entry is outside the destination directory: $entryName" + } + return outputFile + } + + override fun definition() = ModuleDefinition { Name("NativeZipArchive") AsyncFunction("unzip") { sourceFilePath: String, distDirPath: String, promise: Promise -> Thread { + val unzipStartedAtNanos = System.nanoTime() try { + val createdDirectories = mutableSetOf() + val stats = mutableMapOf( + "fileCount" to 0L, + "compressedBytes" to 0L, + "uncompressedBytes" to 0L, + "largestUncompressedEntryBytes" to 0L, + ) + val categories = listOf("NovelAndChapters", "Covers", "NovelFiles", "other") + .associateWith { + mutableMapOf( + "fileCount" to 0L, + "compressedBytes" to 0L, + "uncompressedBytes" to 0L, + ) + } ZipFile(sourceFilePath).use { zis -> zis.entries().asSequence().filterNot { it.isDirectory }.forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val uncompressedBytes = zipEntry.size.coerceAtLeast(0) + val compressedBytes = zipEntry.compressedSize.coerceAtLeast(0) + val firstPathSegment = zipEntry.name.substringBefore('/') + val category = categories[firstPathSegment] ?: categories.getValue("other") + stats["fileCount"] = stats.getValue("fileCount") + 1 + stats["compressedBytes"] = stats.getValue("compressedBytes") + compressedBytes + stats["uncompressedBytes"] = stats.getValue("uncompressedBytes") + uncompressedBytes + stats["largestUncompressedEntryBytes"] = + maxOf(stats.getValue("largestUncompressedEntryBytes"), uncompressedBytes) + category["fileCount"] = category.getValue("fileCount") + 1 + category["compressedBytes"] = category.getValue("compressedBytes") + compressedBytes + category["uncompressedBytes"] = category.getValue("uncompressedBytes") + uncompressedBytes + + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) zis.getInputStream(zipEntry).use { inputStream -> FileOutputStream(newFile).use { fos -> inputStream.copyTo(fos, COPY_BUFFER_SIZE) } } } } - promise.resolve(null) + val result = mutableMapOf() + result.putAll(stats) + result["categories"] = categories + result["elapsedMs"] = + (System.nanoTime() - unzipStartedAtNanos) / 1_000_000.0 + promise.resolve(result) } catch (e: Exception) { promise.reject("UNZIP_FAILED", e.message ?: "Unzip failed", e) } @@ -52,7 +138,39 @@ class NativeZipArchiveModule : Module() { Thread { try { FileOutputStream(zipFilePath).use { fos -> - ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos) } + ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos, excludedFile = File(zipFilePath)) } + } + promise.resolve(null) + } catch (e: Exception) { + promise.reject("ZIP_FAILED", e.message ?: "Zip failed", e) + } + }.start() + } + + AsyncFunction("zipDirectories") { sources: List>, zipFilePath: String, promise: Promise -> + Thread { + try { + require(sources.isNotEmpty()) { "ZIP sources must not be empty" } + require(zipFilePath.isNotBlank()) { "ZIP file path must not be blank" } + val normalizedSources = sources.map { source -> + val sourceDirPath = + source["path"]?.takeIf { it.isNotBlank() } + ?: throw IllegalArgumentException("ZIP source path must not be blank") + require(File(sourceDirPath).isDirectory) { + "ZIP source path is not a directory: $sourceDirPath" + } + val archivePrefix = + source["prefix"]?.let(::normalizeArchivePrefix) + ?: throw IllegalArgumentException("ZIP source prefix is missing") + sourceDirPath to archivePrefix + } + val archiveEntries = mutableSetOf() + FileOutputStream(zipFilePath).use { fos -> + ZipOutputStream(fos).use { zos -> + normalizedSources.forEach { (sourceDirPath, archivePrefix) -> + zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries, File(zipFilePath)) + } + } } promise.resolve(null) } catch (e: Exception) { @@ -69,12 +187,13 @@ class NativeZipArchiveModule : Module() { headers.forEach { (key, value) -> connection.setRequestProperty(key, value) } + val createdDirectories = mutableSetOf() ZipInputStream(connection.inputStream).use { zis -> generateSequence { zis.nextEntry } .filterNot { it.isDirectory } .forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) FileOutputStream(newFile).use { fos -> zis.copyTo(fos, COPY_BUFFER_SIZE) } } } diff --git a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift index 7060acd82a..c36bbd2b6b 100644 --- a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift +++ b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift @@ -13,6 +13,10 @@ public class NativeZipArchiveModule: Module { promise.reject("NOT_IMPLEMENTED", "zip is not implemented on iOS") } + AsyncFunction("zipDirectories") { (sources: [[String: String]], zipFilePath: String, promise: Promise) in + promise.reject("NOT_IMPLEMENTED", "zipDirectories is not implemented on iOS") + } + AsyncFunction("remoteUnzip") { (distDirPath: String, url: String, headers: [String: String], promise: Promise) in promise.reject("NOT_IMPLEMENTED", "remoteUnzip is not implemented on iOS") } diff --git a/modules/native-zip-archive/src/NativeZipArchiveModule.ts b/modules/native-zip-archive/src/NativeZipArchiveModule.ts index cd62ad6fbd..afbe3dfc46 100644 --- a/modules/native-zip-archive/src/NativeZipArchiveModule.ts +++ b/modules/native-zip-archive/src/NativeZipArchiveModule.ts @@ -1,8 +1,27 @@ import { requireNativeModule } from 'expo-modules-core'; +export type UnzipCategoryStats = { + fileCount: number; + compressedBytes: number; + uncompressedBytes: number; +}; + +export type UnzipStats = UnzipCategoryStats & { + largestUncompressedEntryBytes: number; + elapsedMs: number; + categories: Record< + 'NovelAndChapters' | 'Covers' | 'NovelFiles' | 'other', + UnzipCategoryStats + >; +}; + type NativeZipArchiveModule = { - unzip(sourceFilePath: string, distDirPath: string): Promise; + unzip(sourceFilePath: string, distDirPath: string): Promise; zip(sourceDirPath: string, zipFilePath: string): Promise; + zipDirectories( + sources: { path: string; prefix: string }[], + zipFilePath: string, + ): Promise; remoteUnzip( distDirPath: string, urlString: string, diff --git a/src/database/__tests__/db.test.ts b/src/database/__tests__/db.test.ts index 155f67f84d..75a1857e56 100644 --- a/src/database/__tests__/db.test.ts +++ b/src/database/__tests__/db.test.ts @@ -809,6 +809,7 @@ describe('production migrations', () => { '20260719143427_long_moondragon', '20260727081855_calm_chimera', '20260811071655_parched_human_torch', + '20260922120323_demonic_rattler', ]); } finally { sqlite.close(); @@ -1210,4 +1211,67 @@ describe('production migrations', () => { sqlite.close(); } }); + it('discards orphan chapters and preserves the AUTOINCREMENT sequence', async () => { + const sqlite = open({ name: ':memory:' }); + sqlite.executeSync('PRAGMA foreign_keys = ON'); + // Drizzle expects async aliases that op-sqlite's DB type does not declare. + const asyncSqlite = sqlite as unknown as { + executeAsync?: typeof sqlite.execute; + executeRawAsync?: typeof sqlite.executeRaw; + }; + asyncSqlite.executeAsync ??= sqlite.execute; + asyncSqlite.executeRawAsync ??= sqlite.executeRaw; + try { + createSchema(sqlite); + sqlite.executeSync('ALTER TABLE Chapter ADD scanlator text'); + sqlite.executeSync('ALTER TABLE Chapter ADD timeSpent integer DEFAULT 0'); + sqlite.executeSync( + `INSERT INTO Novel (id, path, pluginId, name) + VALUES (1, '/novel', 'plugin', 'Novel')`, + ); + sqlite.executeSync(` + INSERT INTO Chapter (id, novelId, path, name) + VALUES + (1, 1, '/valid', 'Valid'), + (100, 1, '/deleted', 'Deleted') + `); + sqlite.executeSync('DELETE FROM Chapter WHERE id = 100'); + sqlite.executeSync( + `INSERT INTO Chapter (id, novelId, path, name) + VALUES (3, 999, '/orphan', 'Orphan')`, + ); + recordAppliedMigrations(sqlite); + + const drizzleDb = drizzle(sqlite, { schema }); + await migrate(drizzleDb, { + migrations: { + '20260922120323_demonic_rattler': + migrations.migrations['20260922120323_demonic_rattler'], + }, + }); + + expect( + sqlite.executeSync('SELECT id, novelId FROM Chapter').rows, + ).toEqual([{ id: 1, novelId: 1 }]); + expect( + sqlite.executeSync( + "SELECT seq FROM sqlite_sequence WHERE name = 'Chapter'", + ).rows, + ).toEqual([{ seq: 100 }]); + + sqlite.executeSync( + `INSERT INTO Chapter (novelId, path, name) + VALUES (1, '/next', 'Next')`, + ); + expect( + sqlite.executeSync('SELECT last_insert_rowid() AS id').rows, + ).toEqual([{ id: 101 }]); + expect(sqlite.executeRawSync('PRAGMA foreign_key_check;')).toEqual([]); + expect(sqlite.executeRawSync('PRAGMA integrity_check;')).toEqual([ + ['ok'], + ]); + } finally { + sqlite.close(); + } + }); }); diff --git a/src/database/db.ts b/src/database/db.ts index acdd5efcc2..c20d424a96 100644 --- a/src/database/db.ts +++ b/src/database/db.ts @@ -66,6 +66,7 @@ export const drizzleDb = drizzle(_db, { schema, logger: __DEV__ ? new MyLogger() : false, }); +export type DrizzleDb = typeof drizzleDb export const dbManager = createDbManager(drizzleDb); diff --git a/src/database/manager/__tests__/manager.test.ts b/src/database/manager/__tests__/manager.test.ts new file mode 100644 index 0000000000..7f9678670c --- /dev/null +++ b/src/database/manager/__tests__/manager.test.ts @@ -0,0 +1,63 @@ +import { + createTestDb, + cleanupTestDb, +} from '@database/queries/__tests__/testDb'; + +describe('DbManager.executeBatch', () => { + const testDb = createTestDb(); + const flushPendingReactiveQueries = jest.fn(); + + afterAll(() => { + cleanupTestDb(testDb); + }); + + beforeEach(() => { + flushPendingReactiveQueries.mockClear(); + Object.defineProperty(testDb.sqlite, 'flushPendingReactiveQueries', { + configurable: true, + value: flushPendingReactiveQueries, + }); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS BatchTest'); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS MissingBatchTable'); + testDb.sqlite.executeSync( + 'CREATE TABLE BatchTest (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL)', + ); + }); + + it('executes heterogeneous and repeated-parameter commands atomically', async () => { + const result = await testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['first'], ['second']]], + ['UPDATE BatchTest SET value = ? WHERE value = ?', ['updated', 'first']], + ['DELETE FROM BatchTest WHERE value = ?', [['second']]], + ]); + + expect(result.rowsAffected).toBe(4); + expect(flushPendingReactiveQueries).toHaveBeenCalledTimes(1); + expect( + testDb.sqlite.executeSync('SELECT value FROM BatchTest').rows, + ).toEqual([{ value: 'updated' }]); + }); + + it('rolls back every command when a later command fails', async () => { + await expect( + testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['rolled back']]], + ['INSERT INTO MissingBatchTable (value) VALUES (?)', [['failure']]], + ]), + ).rejects.toMatchObject({ + message: expect.stringContaining('MissingBatchTable'), + }); + + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + expect(testDb.sqlite.executeSync('SELECT * FROM BatchTest').rows).toEqual( + [], + ); + }); + + it('does not flush for an empty batch', async () => { + const result = await testDb.dbManager.executeBatch([]); + + expect(result).toEqual({ rowsAffected: 0 }); + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + }); +}); diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 195bab40bb..95584c8c0d 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -1,27 +1,30 @@ // db-manager.types.ts import type { - SQLiteTransaction, - TablesRelationalConfig, - Placeholder, -} from 'drizzle-orm'; -import { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; + BatchQueryResult, + SQLBatchTuple, +} from '@op-engineering/op-sqlite'; +import type { DrizzleDb } from '@database/db'; +import type { Placeholder } from 'drizzle-orm'; +import type { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; // Define the TransactionParameter type based on your DrizzleDb -export type TransactionParameter = SQLiteTransaction< - 'async', - { changes: number; lastInsertRowid: number } | void, - Record, - TablesRelationalConfig ->; - +export type TransactionParameter = Parameters< + Parameters[0] +>[0]; /** * Interface defining the public API and documentation for the Drizzle database manager. * This contract ensures consistent documentation and type safety across the application. */ export interface IDbManager { + /** + * Executes raw op-sqlite commands as one atomic transaction. + */ + executeBatch(commands: SQLBatchTuple[]): Promise; /** * Efficiently executes a Drizzle query for multiple data rows using - * op-sqlite executeBatch under the hood. + * op-sqlite executeBatch under the hood. The batch owns its transaction; + * it cannot be nested inside write(). The callback argument is only used + * to build the prepared query and is not the surrounding write transaction. */ batch>( data: T[], @@ -29,7 +32,7 @@ export interface IDbManager { tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ): Promise; + ): Promise; /** * Creates a subquery that defines a temporary named result set as a CTE. diff --git a/src/database/manager/manager.ts b/src/database/manager/manager.ts index cc5690d755..6ce98c6ef2 100644 --- a/src/database/manager/manager.ts +++ b/src/database/manager/manager.ts @@ -1,5 +1,9 @@ +import type { + BatchQueryResult, + SQLBatchTuple, + Scalar, +} from '@op-engineering/op-sqlite'; import type { drizzleDb } from '@database/db'; -import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; import { IDbManager } from './manager.d'; import { DbTaskQueue } from './queue'; import { Schema } from '../schema'; @@ -88,15 +92,32 @@ class DbManager implements IDbManager { .rows as Awaited>; } + public async executeBatch( + commands: SQLBatchTuple[], + ): Promise { + if (!commands.length) { + return { rowsAffected: 0 }; + } + + return await this.queue.enqueue({ + id: 'write', + run: async () => { + const result = await this.db.$client.executeBatch(commands); + this.db.$client.flushPendingReactiveQueries(); + return result; + }, + }); + } + public async batch>( data: T[], fn: ( tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ) { + ): Promise { if (!data.length) { - return; + return { rowsAffected: 0 }; } const ph = (arg: Extract) => sql.placeholder(arg); @@ -110,13 +131,7 @@ class DbManager implements IDbManager { }); const commands: SQLBatchTuple[] = [[query.sql, params]]; - await this.queue.enqueue({ - id: 'write', - run: async () => { - await this.db.$client.executeBatch(commands); - this.db.$client?.flushPendingReactiveQueries(); - }, - }); + return await this.executeBatch(commands); } public async write( diff --git a/src/database/queries/CategoryQueries.ts b/src/database/queries/CategoryQueries.ts index 2894c88888..54562a9b88 100644 --- a/src/database/queries/CategoryQueries.ts +++ b/src/database/queries/CategoryQueries.ts @@ -269,16 +269,23 @@ export const _restoreCategory = async ( categoryId = restoredCategory.id; } - // Insert novel-category associations if (category.novelIds && category.novelIds.length > 0) { - for (const backupNovelId of category.novelIds) { - const novelId = novelIdMap?.get(backupNovelId) ?? backupNovelId; + const membershipBatchSize = 500; + for ( + let batchStart = 0; + batchStart < category.novelIds.length; + batchStart += membershipBatchSize + ) { + const memberships = category.novelIds + .slice(batchStart, batchStart + membershipBatchSize) + .map(backupNovelId => ({ + categoryId, + novelId: novelIdMap?.get(backupNovelId) ?? backupNovelId, + })); + await tx .insert(novelCategorySchema) - .values({ - categoryId, - novelId: novelId, - }) + .values(memberships) .onConflictDoNothing() .run(); } diff --git a/src/database/queries/ChapterQueries.ts b/src/database/queries/ChapterQueries.ts index 9b7ca28cf4..3bcddc945b 100644 --- a/src/database/queries/ChapterQueries.ts +++ b/src/database/queries/ChapterQueries.ts @@ -468,14 +468,24 @@ export const getNovelChapters = async ( * a paginated UI query because doing so silently produces incomplete backups. */ export const getAllNovelChaptersForBackup = async ( - novelId: number, -): Promise => - dbManager + novelIds: number | number[], +): Promise => { + const ids = Array.isArray(novelIds) ? novelIds : [novelIds]; + if (ids.length === 0) { + return []; + } + + return dbManager .select() .from(chapterSchema) - .where(eq(chapterSchema.novelId, novelId)) - .orderBy(asc(chapterSchema.id)) + .where( + Array.isArray(novelIds) + ? inArray(chapterSchema.novelId, ids) + : eq(chapterSchema.novelId, ids[0]), + ) + .orderBy(asc(chapterSchema.novelId), asc(chapterSchema.id)) .all(); +}; export const getNovelChaptersSync = ( novelId: number, diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index 3d66c4d298..59c6cfc212 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -6,12 +6,7 @@ import { insertChapters } from './ChapterQueries'; import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; -import { - BackupNovel, - DBNovelInfo, - NovelInfo, - type RestoredNovelMapping, -} from '../types'; +import { DBNovelInfo, NovelInfo } from '../types'; import { SourceNovel } from '@plugins/types'; import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; @@ -21,7 +16,6 @@ import { novelSchema, novelCategorySchema, categorySchema, - chapterSchema, } from '@database/schema'; import type { TransactionParameter } from '@database/manager/manager.d'; import { getLibraryDefaultCategoryId } from '@hooks/persisted/useSettings'; @@ -75,7 +69,7 @@ const getCategoriesForNewNovel = async ( return getBuiltInDefaultCategory(tx); }; -const getCategoryForNewNovel = async (tx: TransactionParameter) => +export const getCategoryForNewNovel = async (tx: TransactionParameter) => (await getCategoriesForNewNovel(tx))[0]; /** @@ -287,69 +281,6 @@ export const deleteCachedNovels = async () => { showToast(getString('advancedSettingsScreen.cachedNovelsDeletedToast')); }; -/** - * Restore a novel from backup using Drizzle ORM. - */ -export const restoreLibrary = async (novel: NovelInfo) => { - const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { - throw e; - }); - - const novelId = await dbManager.write(async tx => { - const row = await tx - .insert(novelSchema) - .values({ - path: sourceNovel.path, - name: novel.name, - pluginId: novel.pluginId, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - name: novel.name, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }, - }) - .returning() - .get(); - - if (row) { - const defaultCategory = await getCategoryForNewNovel(tx); - - if (defaultCategory) { - await tx - .insert(novelCategorySchema) - .values({ - novelId: row.id, - categoryId: defaultCategory.id, - }) - .onConflictDoNothing() - .run(); - } - } - return row?.id; - }); - - if (novelId && sourceNovel.chapters) { - await insertChapters(novelId, sourceNovel.chapters); - } -}; - export const updateNovelInfo = async (info: NovelInfo) => { await dbManager.write(async tx => { await tx @@ -474,89 +405,3 @@ export const updateNovelCategories = async ( } }); }; - -/** - * Restores novel and chapters from a backup object. - */ -export const _restoreNovelAndChapters = async ( - backupNovel: BackupNovel, -): Promise => { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - return dbManager.write(async tx => { - // Match novels by their stable source identity, not the database-local ID. - const restoredNovel = await tx - .insert(novelSchema) - .values({ - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }, - }) - .returning({ id: novelSchema.id }) - .get(); - - if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { - const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; - await tx - .update(novelSchema) - .set({ - cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); - } - - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); - - const chapterMappings: RestoredNovelMapping['chapters'] = []; - - // Restore chapters in batches - if (chapters.length > 0) { - const BATCH_SIZE = 100; - for (let i = 0; i < chapters.length; i += BATCH_SIZE) { - const batch = chapters.slice(i, i + BATCH_SIZE); - const restoredChapters = await tx - .insert(chapterSchema) - .values( - batch.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId: restoredNovel.id, - })), - ) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all(); - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } - } - - return { - pluginId: novel.pluginId, - backupNovelId, - restoredNovelId: restoredNovel.id, - chapters: chapterMappings, - }; - }); -}; diff --git a/src/database/queries/NovelRestoreQueries.ts b/src/database/queries/NovelRestoreQueries.ts new file mode 100644 index 0000000000..89a34afae4 --- /dev/null +++ b/src/database/queries/NovelRestoreQueries.ts @@ -0,0 +1,774 @@ +import { and, eq, inArray, or } from 'drizzle-orm'; +import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; + +import { fetchNovel } from '@services/plugin/fetch'; +import { insertChapters } from './ChapterQueries'; +import { getCategoryForNewNovel } from './NovelQueries'; + +import { dbManager } from '@database/db'; +import { + createNovelTriggerQueryDelete, + createNovelTriggerQueryInsert, + createNovelTriggerQueryUpdate, +} from '@database/queryStrings/triggers'; +import { + novelCategorySchema, + novelSchema, + restoreChapterMappingSchema, +} from '@database/schema'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import type { BackupNovel, NovelInfo, RestoredNovelMapping } from '../types'; + +/** + * Restore a novel from backup using Drizzle ORM. + */ +export const restoreLibrary = async (novel: NovelInfo) => { + const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { + throw e; + }); + + const novelId = await dbManager.write(async tx => { + const row = await tx + .insert(novelSchema) + .values({ + path: sourceNovel.path, + name: novel.name, + pluginId: novel.pluginId, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: { + name: novel.name, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }, + }) + .returning() + .get(); + + if (row) { + const defaultCategory = await getCategoryForNewNovel(tx); + + if (defaultCategory) { + await tx + .insert(novelCategorySchema) + .values({ + novelId: row.id, + categoryId: defaultCategory.id, + }) + .onConflictDoNothing() + .run(); + } + } + return row?.id; + }); + + if (novelId && sourceNovel.chapters) { + await insertChapters(novelId, sourceNovel.chapters); + } +}; +const sqliteBoolean = (value: boolean | null | undefined): Scalar => + value == null ? null : value ? 1 : 0; +const RESTORE_NOVEL_BATCH_SIZE = 100; +const RESTORE_CHAPTER_BATCH_SIZE = 10_000; + +export type RestoreNovelProgress = { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; +}; + +export type RestoreNovelOptions = { + includeChapterMappings?: boolean; + restoreRunId?: string; + onProgress?: (progress: RestoreNovelProgress) => void; +}; + +export type RestoreNovelMetrics = { + novelUpsertCalls: number; + novelUpsertRowsAttempted: number; + novelUpsertMs: number; + novelIdentityLookupCalls: number; + novelIdentityLookupRowsReturned: number; + novelIdentityLookupMs: number; + chapterWriteChunkCalls: number; + chapterWriteRowsAttempted: number; + chapterWriteMs: number; + chapterMappingRowsAttempted: number; + statsRefreshCalls: number; + statsRefreshNovels: number; + statsRefreshMs: number; + novelBatchFallbacks: number; + novelFallbackRowsAttempted: number; + chapterBatchFallbacks: number; + chapterFallbackRowsAttempted: number; + fallbackCauses: { name: string; message: string }[]; +}; + +export const recordRestoreFallback = ( + metrics: RestoreNovelMetrics | undefined, + error: unknown, +) => { + if (!metrics || metrics.fallbackCauses.length >= 5) { + return; + } + + const errorName = + error instanceof Error && + [ + 'AggregateError', + 'DatabaseError', + 'Error', + 'EvalError', + 'ReferenceError', + 'RangeError', + 'SQLiteError', + 'SqliteError', + 'SyntaxError', + 'TypeError', + 'URIError', + ].includes(error.name) + ? error.name + : 'Error'; + const message = + error instanceof Error && /constraint/i.test(error.message) + ? 'Database constraint failed' + : error instanceof Error && /locked|busy/i.test(error.message) + ? 'Database is busy' + : error instanceof Error && /no such table/i.test(error.message) + ? 'Database table missing' + : error instanceof Error && /no such column/i.test(error.message) + ? 'Database column missing' + : 'Database operation failed'; + metrics.fallbackCauses.push({ name: errorName, message }); +}; + +const createPlaceholderRows = (rowCount: number, valuesPerRow: number) => + Array.from( + { length: rowCount }, + () => `(${Array(valuesPerRow).fill('?').join(', ')})`, + ).join(',\n'); + +const buildNovelUpsertSql = (rowCount: number) => ` + INSERT INTO Novel ( + path, pluginId, name, cover, summary, author, artist, status, genres, + inLibrary, isLocal, totalPages + ) VALUES ${createPlaceholderRows(rowCount, 12)} + ON CONFLICT(path, pluginId) DO UPDATE SET + path = excluded.path, + pluginId = excluded.pluginId, + name = excluded.name, + cover = excluded.cover, + summary = excluded.summary, + author = excluded.author, + artist = excluded.artist, + status = excluded.status, + genres = excluded.genres, + inLibrary = excluded.inLibrary, + isLocal = excluded.isLocal, + totalPages = excluded.totalPages +`; + +const NOVEL_UPSERT_SQL = buildNovelUpsertSql(RESTORE_NOVEL_BATCH_SIZE); + +const NOVEL_COVER_UPDATE_SQL = ` + UPDATE Novel + SET cover = ? || pluginId || '/' || id || '/cover.png' || ? + WHERE pluginId = ? AND path = ? +`; + +const restoreNovelValues = (novel: BackupNovel): Scalar[] => [ + novel.path, + novel.pluginId, + novel.name, + novel.cover ?? null, + novel.summary ?? null, + novel.author ?? null, + novel.artist ?? null, + novel.status ?? null, + novel.genres ?? null, + sqliteBoolean(novel.inLibrary), + sqliteBoolean(novel.isLocal), + novel.totalPages ?? null, +]; + +const restoreNovelChunk = async ( + backupNovels: BackupNovel[], + metrics?: RestoreNovelMetrics, +): Promise => { + const commands: SQLBatchTuple[] = [ + [ + backupNovels.length === RESTORE_NOVEL_BATCH_SIZE + ? NOVEL_UPSERT_SQL + : buildNovelUpsertSql(backupNovels.length), + backupNovels.flatMap(restoreNovelValues), + ], + ]; + const storedCovers = backupNovels + .filter(novel => novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) + .map( + novel => + [ + `file://${NOVEL_STORAGE}/`, + novel.cover?.match(/[?#].*$/)?.[0] ?? '', + novel.pluginId, + novel.path, + ] as Scalar[], + ); + if (storedCovers.length > 0) { + commands.push([NOVEL_COVER_UPDATE_SQL, storedCovers]); + } + const upsertStartedAt = metrics ? performance.now() : 0; + if (metrics) { + metrics.novelUpsertCalls++; + metrics.novelUpsertRowsAttempted += backupNovels.length; + } + try { + await dbManager.executeBatch(commands); + } finally { + if (metrics) { + metrics.novelUpsertMs += performance.now() - upsertStartedAt; + } + } + + if (metrics) { + metrics.novelIdentityLookupCalls++; + } + const lookupStartedAt = metrics ? performance.now() : 0; + let rows; + try { + rows = await dbManager + .select({ + id: novelSchema.id, + path: novelSchema.path, + pluginId: novelSchema.pluginId, + }) + .from(novelSchema) + .where( + or( + ...backupNovels.map(novel => + and( + eq(novelSchema.pluginId, novel.pluginId), + eq(novelSchema.path, novel.path), + ), + ), + ), + ) + .all(); + if (metrics) { + metrics.novelIdentityLookupRowsReturned += rows.length; + } + } finally { + if (metrics) { + metrics.novelIdentityLookupMs += performance.now() - lookupStartedAt; + } + } + const rowsByIdentity = new Map( + rows.map(row => [`${row.pluginId}\u0000${row.path}`, row]), + ); + return backupNovels.map(backupNovel => { + const row = rowsByIdentity.get( + `${backupNovel.pluginId}\u0000${backupNovel.path}`, + ); + if (!row) { + throw new Error('Restore returned incomplete novel mapping'); + } + return { + pluginId: backupNovel.pluginId, + backupNovelId: backupNovel.id, + restoredNovelId: row.id, + }; + }); +}; + +const restoreNovelChunkWithRetry = async ( + backupNovels: BackupNovel[], + metrics?: RestoreNovelMetrics, +): Promise => { + try { + return await restoreNovelChunk(backupNovels, metrics); + } catch (error) { + if (metrics) { + metrics.novelBatchFallbacks++; + } + recordRestoreFallback(metrics, error); + const mappings: RestoredNovelMapping[] = []; + let failed = false; + for (const backupNovel of backupNovels) { + if (metrics) { + metrics.novelFallbackRowsAttempted++; + } + try { + mappings.push( + await restoreNovelChunk([backupNovel], metrics).then(([mapping]) => { + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; + }), + ); + } catch (rowError) { + recordRestoreFallback(metrics, rowError); + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more novels'); + } + return mappings; + } +}; + +const appendRestoreChapterValues = ( + values: Scalar[], + chapter: BackupNovel['chapters'][number], + restoredNovelId: number, +) => { + values.push( + restoredNovelId, + chapter.path, + chapter.name, + chapter.releaseTime ?? null, + sqliteBoolean(chapter.bookmark), + sqliteBoolean(chapter.unread), + chapter.readTime ?? null, + sqliteBoolean(chapter.isDownloaded), + chapter.updatedTime ?? null, + chapter.chapterNumber ?? null, + chapter.page ?? null, + chapter.position ?? null, + chapter.progress ?? null, + chapter.scanlator ?? null, + chapter.timeSpent ?? null, + ); +}; + +type ChapterRestoreRecord = { + backupNovelId: number; + backupChapterId: number; + restoredNovelId: number; + chapter: BackupNovel['chapters'][number]; +}; + +const RESTORE_CHAPTER_STATEMENT_SIZE = 100; + +const buildChapterUpsertSql = (rowCount: number) => ` + INSERT INTO Chapter ( + novelId, path, name, releaseTime, bookmark, unread, readTime, + isDownloaded, updatedTime, chapterNumber, page, position, progress, + scanlator, timeSpent + ) VALUES ${createPlaceholderRows(rowCount, 15)} + ON CONFLICT(novelId, path) DO UPDATE SET + novelId = excluded.novelId, + path = excluded.path, + name = excluded.name, + releaseTime = excluded.releaseTime, + bookmark = excluded.bookmark, + unread = excluded.unread, + readTime = excluded.readTime, + isDownloaded = excluded.isDownloaded, + updatedTime = excluded.updatedTime, + chapterNumber = excluded.chapterNumber, + page = excluded.page, + position = excluded.position, + progress = excluded.progress, + scanlator = excluded.scanlator, + timeSpent = excluded.timeSpent +`; + +const CHAPTER_UPSERT_SQL = buildChapterUpsertSql( + RESTORE_CHAPTER_STATEMENT_SIZE, +); + +const buildRestoreChapterMappingInsertSql = (rowCount: number) => ` + WITH restoreInputs( + ordinal, restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, chapterPath + ) AS ( + VALUES ${Array.from( + { length: rowCount }, + (_, ordinal) => `(${ordinal}, ?, ?, ?, ?, ?)`, + ).join(',\n')} + ) + INSERT INTO RestoreChapterMapping ( + restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, restoredChapterId + ) + SELECT + restoreInputs.restoreRunId, + restoreInputs.backupNovelId, + restoreInputs.backupChapterId, + restoreInputs.restoredNovelId, + chapter.id + FROM restoreInputs + JOIN Chapter AS chapter + ON chapter.novelId = restoreInputs.restoredNovelId + AND chapter.path = restoreInputs.chapterPath + WHERE 1 + ORDER BY restoreInputs.ordinal + ON CONFLICT(restoreRunId, backupNovelId, backupChapterId) DO UPDATE SET + restoredNovelId = excluded.restoredNovelId, + restoredChapterId = excluded.restoredChapterId +`; +const RESTORE_CHAPTER_MAPPING_INSERT_SQL = buildRestoreChapterMappingInsertSql( + RESTORE_CHAPTER_STATEMENT_SIZE, +); + +const restoreChapterChunk = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, + metrics?: RestoreNovelMetrics, +) => { + if (records.length === 0) { + return; + } + if (includeChapterMappings && !restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const commands: SQLBatchTuple[] = [ + ['DROP TRIGGER IF EXISTS update_novel_stats'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_update'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_delete'], + ]; + for ( + let start = 0; + start < records.length; + start += RESTORE_CHAPTER_STATEMENT_SIZE + ) { + const rowCount = Math.min( + RESTORE_CHAPTER_STATEMENT_SIZE, + records.length - start, + ); + const values: Scalar[] = []; + for (let index = start; index < start + rowCount; index++) { + const record = records[index]; + appendRestoreChapterValues( + values, + record.chapter, + record.restoredNovelId, + ); + } + const chapterUpsertSql = + rowCount === RESTORE_CHAPTER_STATEMENT_SIZE + ? CHAPTER_UPSERT_SQL + : buildChapterUpsertSql(rowCount); + commands.push([chapterUpsertSql, values]); + } + if (includeChapterMappings) { + for ( + let start = 0; + start < records.length; + start += RESTORE_CHAPTER_STATEMENT_SIZE + ) { + const rowCount = Math.min( + RESTORE_CHAPTER_STATEMENT_SIZE, + records.length - start, + ); + const values: Scalar[] = []; + for (let index = start; index < start + rowCount; index++) { + const record = records[index]; + values.push( + restoreRunId as string, + record.backupNovelId, + record.backupChapterId, + record.restoredNovelId, + record.chapter.path, + ); + } + const mappingInsertSql = + rowCount === RESTORE_CHAPTER_STATEMENT_SIZE + ? RESTORE_CHAPTER_MAPPING_INSERT_SQL + : buildRestoreChapterMappingInsertSql(rowCount); + commands.push([mappingInsertSql, values]); + } + } + commands.push( + [createNovelTriggerQueryInsert], + [createNovelTriggerQueryDelete], + [createNovelTriggerQueryUpdate], + ); + if (metrics) { + metrics.chapterWriteChunkCalls++; + metrics.chapterWriteRowsAttempted += records.length; + if (includeChapterMappings) { + metrics.chapterMappingRowsAttempted += records.length; + } + } + const chapterWriteStartedAt = metrics ? performance.now() : 0; + try { + await dbManager.executeBatch(commands); + } finally { + if (metrics) { + metrics.chapterWriteMs += performance.now() - chapterWriteStartedAt; + } + } +}; + +const restoreChapterChunkWithRetry = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, + metrics?: RestoreNovelMetrics, +) => { + try { + await restoreChapterChunk( + records, + includeChapterMappings, + restoreRunId, + metrics, + ); + } catch (error) { + if (metrics) { + metrics.chapterBatchFallbacks++; + } + recordRestoreFallback(metrics, error); + let failed = false; + for (const record of records) { + if (metrics) { + metrics.chapterFallbackRowsAttempted++; + } + try { + await restoreChapterChunk( + [record], + includeChapterMappings, + restoreRunId, + metrics, + ); + } catch (rowError) { + recordRestoreFallback(metrics, rowError); + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more chapters'); + } + } +}; + +const NOVEL_STATS_UPDATE_SQL = ` + UPDATE Novel + SET totalChapters = ( + SELECT COUNT(*) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + chaptersDownloaded = COALESCE(( + SELECT SUM(CASE WHEN Chapter.isDownloaded = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + chaptersUnread = COALESCE(( + SELECT SUM(CASE WHEN Chapter.unread = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + lastReadAt = ( + SELECT MAX(Chapter.readTime) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + lastUpdatedAt = ( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Novel.id + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC + LIMIT 1 + ) + WHERE Novel.id = ? +`; + +const refreshRestoredNovelStats = async ( + novelIds: number[], + metrics?: RestoreNovelMetrics, +) => { + if (novelIds.length === 0) { + return; + } + if (metrics) { + metrics.statsRefreshCalls++; + metrics.statsRefreshNovels += novelIds.length; + } + const statsRefreshStartedAt = metrics ? performance.now() : 0; + try { + await dbManager.executeBatch([ + [NOVEL_STATS_UPDATE_SQL, novelIds.map(id => [id] as Scalar[])], + ]); + } finally { + if (metrics) { + metrics.statsRefreshMs += performance.now() - statsRefreshStartedAt; + } + } +}; + +export const _restoreNovelsAndChapters = async ( + backupNovels: BackupNovel[], + options: RestoreNovelOptions = {}, + metrics?: RestoreNovelMetrics, +): Promise => { + if (backupNovels.length === 0) { + return []; + } + const includeChapterMappings = options.includeChapterMappings ?? true; + if (includeChapterMappings && !options.restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const mappings: RestoredNovelMapping[] = []; + for ( + let start = 0; + start < backupNovels.length; + start += RESTORE_NOVEL_BATCH_SIZE + ) { + const chunkMappings = await restoreNovelChunkWithRetry( + backupNovels.slice(start, start + RESTORE_NOVEL_BATCH_SIZE), + metrics, + ); + mappings.push(...chunkMappings); + options.onProgress?.({ + stage: 'novels', + completed: mappings.length, + total: backupNovels.length, + }); + } + + const restoredNovelIds = new Map( + mappings.map(mapping => [mapping.backupNovelId, mapping.restoredNovelId]), + ); + const totalChapters = backupNovels.reduce( + (total, novel) => total + novel.chapters.length, + 0, + ); + let completedChapters = 0; + const chapterChunk: ChapterRestoreRecord[] = []; + for (const backupNovel of backupNovels) { + const restoredNovelId = restoredNovelIds.get(backupNovel.id); + if (restoredNovelId === undefined) { + throw new Error('Missing restored novel mapping'); + } + for (const chapter of backupNovel.chapters) { + chapterChunk.push({ + backupNovelId: backupNovel.id, + backupChapterId: chapter.id, + restoredNovelId, + chapter, + }); + if (chapterChunk.length === RESTORE_CHAPTER_BATCH_SIZE) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + metrics, + ); + completedChapters += chapterChunk.length; + chapterChunk.length = 0; + options.onProgress?.({ + stage: 'chapters', + completed: completedChapters, + total: totalChapters, + }); + } + } + } + if (chapterChunk.length > 0) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + metrics, + ); + completedChapters += chapterChunk.length; + options.onProgress?.({ + stage: 'chapters', + completed: completedChapters, + total: totalChapters, + }); + } else if (totalChapters === 0) { + options.onProgress?.({ + stage: 'chapters', + completed: 0, + total: 0, + }); + } + const restoredIds = [...restoredNovelIds.values()]; + options.onProgress?.({ + stage: 'stats', + completed: 0, + total: restoredIds.length, + }); + await refreshRestoredNovelStats(restoredIds, metrics); + options.onProgress?.({ + stage: 'stats', + completed: restoredIds.length, + total: restoredIds.length, + }); + return mappings; +}; + +/** + * Restores a novel and its chapters from a backup object. + */ +export const _restoreNovelAndChapters = async ( + backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, + metrics?: RestoreNovelMetrics, +): Promise => { + const [mapping] = await _restoreNovelsAndChapters( + [backupNovel], + options, + metrics, + ); + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; +}; + +export const getRestoreChapterMappings = async ( + restoreRunId: string, + backupNovelId: number, + backupChapterIds: number[], +) => { + if (backupChapterIds.length === 0) { + return []; + } + return dbManager + .select({ + backupChapterId: restoreChapterMappingSchema.backupChapterId, + restoredChapterId: restoreChapterMappingSchema.restoredChapterId, + }) + .from(restoreChapterMappingSchema) + .where( + and( + eq(restoreChapterMappingSchema.restoreRunId, restoreRunId), + eq(restoreChapterMappingSchema.backupNovelId, backupNovelId), + inArray(restoreChapterMappingSchema.backupChapterId, backupChapterIds), + ), + ) + .all(); +}; + +export const clearRestoreChapterMappings = async (restoreRunId: string) => { + await dbManager.write(tx => + tx + .delete(restoreChapterMappingSchema) + .where(eq(restoreChapterMappingSchema.restoreRunId, restoreRunId)) + .run(), + ); +}; diff --git a/src/database/queries/__tests__/CategoryQueries.test.ts b/src/database/queries/__tests__/CategoryQueries.test.ts index ae3cb2f5eb..ceb3c58e86 100644 --- a/src/database/queries/__tests__/CategoryQueries.test.ts +++ b/src/database/queries/__tests__/CategoryQueries.test.ts @@ -12,7 +12,11 @@ import { insertTestNovelCategory, clearAllTables, } from './testData'; -import { categorySchema, novelCategorySchema } from '@database/schema'; +import { + categorySchema, + novelCategorySchema, + novelSchema, +} from '@database/schema'; import { eq, sql } from 'drizzle-orm'; import { @@ -86,6 +90,56 @@ describe('CategoryQueries', () => { .map(membership => membership.novelId), ).toEqual(expect.arrayContaining([existingNovelId, restoredNovelId])); }); + it('restores memberships in bounded batches with remapping and conflict-ignore semantics', async () => { + const testDb = getTestDb(); + const existingNovelId = await insertTestNovel(testDb, { + inLibrary: true, + }); + const secondNovelId = await insertTestNovel(testDb, { + inLibrary: true, + }); + const novelIds = [ + ...Array.from({ length: 501 }, (_, index) => index + 1), + 1, + 501, + 987654321, + ]; + const novelIdMap = new Map([ + [1, existingNovelId], + [2, secondNovelId], + ]); + const categoryId = await _restoreCategory( + { + id: 3, + name: 'Batch restored', + sort: 3, + novelIds, + }, + novelIdMap, + ); + + const memberships = await testDb.drizzleDb + .select({ novelId: novelCategorySchema.novelId }) + .from(novelCategorySchema) + .where(eq(novelCategorySchema.categoryId, categoryId)) + .all(); + const membershipIds = memberships.map(membership => membership.novelId); + const expectedMembershipIds = Array.from( + new Set(novelIds.map(id => novelIdMap.get(id) ?? id)), + ).sort((a, b) => a - b); + + expect(membershipIds.sort((a, b) => a - b)).toEqual( + expectedMembershipIds, + ); + expect( + await testDb.drizzleDb + .select({ id: novelSchema.id }) + .from(novelSchema) + .where(eq(novelSchema.id, 987654321)) + .get(), + ).toBeUndefined(); + expect(membershipIds).toContain(987654321); + }); it('does not overwrite a custom category with a colliding backup ID', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index 687f833db3..263d790888 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -153,9 +153,7 @@ describe('ChapterQueries', () => { describe('getAllNovelChaptersForBackup', () => { it('returns chapters beyond the 1000-row UI query limit', async () => { const testDb = getTestDb(); - // No Novel row is needed for this selector test. Keeping the id orphaned - // also avoids running aggregate-stat triggers 1001 times during setup. - const novelId = 123456; + const novelId = await insertTestNovel(testDb); const values = Array.from( { length: 1001 }, @@ -185,6 +183,33 @@ describe('ChapterQueries', () => { expect(backupChapters).toHaveLength(1001); expect(backupChapters.at(-1)?.name).toBe('Chapter 1001'); }); + + it('returns chapters for multiple novels in novel and chapter order', async () => { + const testDb = getTestDb(); + const firstNovelId = await insertTestNovel(testDb); + const secondNovelId = await insertTestNovel(testDb); + testDb.sqlite.executeSync(` + INSERT INTO Chapter + (novelId, path, name, chapterNumber, page, position) + VALUES + (${firstNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${firstNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2), + (${secondNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${secondNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2) + `); + + const chapters = await getAllNovelChaptersForBackup([ + secondNovelId, + firstNovelId, + ]); + + expect(chapters.map(chapter => [chapter.novelId, chapter.name])).toEqual([ + [firstNovelId, 'Chapter 1'], + [firstNovelId, 'Chapter 2'], + [secondNovelId, 'Chapter 1'], + [secondNovelId, 'Chapter 2'], + ]); + }); }); describe('markChaptersRead', () => { diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index 916daad39b..7185332849 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -14,7 +14,6 @@ import { } from './testData'; import { categorySchema, - chapterSchema, novelCategorySchema, novelSchema, } from '@database/schema'; @@ -30,12 +29,10 @@ import { removeNovelsFromLibrary, getCachedNovels, deleteCachedNovels, - restoreLibrary, updateNovelInfo, pickCustomNovelCover, updateNovelCategoryById, updateNovelCategories, - _restoreNovelAndChapters, } from '../NovelQueries'; const mockGetLibraryDefaultCategoryId = jest.fn(); @@ -421,112 +418,6 @@ describe('NovelQueries', () => { }); }); - describe('restoreLibrary', () => { - it('should restore novel from backup', async () => { - const novel = { - id: 999, - path: '/test/novel', - pluginId: 'test-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 1, - chaptersDownloaded: 0, - chaptersUnread: 0, - totalChapters: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; - - // Mock fetchNovel to return a valid SourceNovel - const { fetchNovel } = require('@services/plugin/fetch'); - jest.mocked(fetchNovel).mockResolvedValueOnce({ - id: undefined, - path: '/test/novel', - name: 'Restored Novel', - chapters: [], - }); - - await restoreLibrary(novel); - - const restored = await getNovelByPath('/test/novel', 'test-plugin'); - expect(restored?.name).toBe('Restored Novel'); - }); - }); - - describe('_restoreNovelAndChapters', () => { - it('does not replace an unrelated novel when backup IDs collide', async () => { - const testDb = getTestDb(); - await insertTestNovel(testDb, { - path: '/existing/novel', - pluginId: 'existing-plugin', - name: 'Existing Novel', - inLibrary: true, - }); - - const mapping = await _restoreNovelAndChapters({ - id: 1, - path: '/restored/novel', - pluginId: 'restored-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 10, - novelId: 1, - path: '/restored/chapter-1', - name: 'Chapter 1', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: null, - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }); - - expect(mapping.restoredNovelId).not.toBe(1); - expect( - (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, - ).toBe('Existing Novel'); - expect( - (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, - ).toBe(mapping.restoredNovelId); - const restoredChapters = await testDb.drizzleDb - .select() - .from(chapterSchema) - .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) - .all(); - expect(mapping.chapters).toEqual([ - { - backupChapterId: 10, - restoredChapterId: restoredChapters[0].id, - }, - ]); - }); - }); - describe('updateNovelInfo', () => { it('should update novel information', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/NovelRestoreQueries.test.ts b/src/database/queries/__tests__/NovelRestoreQueries.test.ts new file mode 100644 index 0000000000..e33d7bad23 --- /dev/null +++ b/src/database/queries/__tests__/NovelRestoreQueries.test.ts @@ -0,0 +1,1035 @@ +import './mockDb'; +import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; +import { insertTestNovel, insertTestChapter, clearAllTables } from './testData'; +import { chapterSchema, novelSchema } from '@database/schema'; +import { and, eq } from 'drizzle-orm'; +import type { SQLBatchTuple } from '@op-engineering/op-sqlite'; +import type { BackupNovel, ChapterInfo } from '../../types'; + +import { getNovelByPath } from '../NovelQueries'; +import { + clearRestoreChapterMappings, + getRestoreChapterMappings, + restoreLibrary, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '../NovelRestoreQueries'; + +const mockGetLibraryDefaultCategoryId = jest.fn(); + +jest.mock('@hooks/persisted/useSettings', () => ({ + getLibraryDefaultCategoryId: () => mockGetLibraryDefaultCategoryId(), +})); + +const createBackupChapter = ( + id: number, + novelId: number, + path: string, + values: Partial = {}, +): ChapterInfo => ({ + id, + novelId, + path, + name: `Chapter ${id}`, + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + ...values, +}); + +const createBackupNovel = ( + id: number, + path: string, + chapters: ChapterInfo[] = [], + pluginId = 'restore-plugin', +): BackupNovel => ({ + id, + path, + pluginId, + name: `Novel ${id}`, + cover: null, + summary: null, + author: null, + artist: null, + status: 'Unknown', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters, +}); + +describe('NovelRestoreQueries', () => { + beforeEach(() => { + const testDb = setupTestDatabase(); + clearAllTables(testDb); + mockGetLibraryDefaultCategoryId.mockReturnValue(undefined); + }); + + afterAll(() => { + teardownTestDatabase(); + }); + describe('restoreLibrary', () => { + it('should restore novel from backup', async () => { + const novel = { + id: 999, + path: '/test/novel', + pluginId: 'test-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 1, + chaptersDownloaded: 0, + chaptersUnread: 0, + totalChapters: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // Mock fetchNovel to return a valid SourceNovel + const { fetchNovel } = require('@services/plugin/fetch'); + jest.mocked(fetchNovel).mockResolvedValueOnce({ + id: undefined, + path: '/test/novel', + name: 'Restored Novel', + chapters: [], + }); + + await restoreLibrary(novel); + + const restored = await getNovelByPath('/test/novel', 'test-plugin'); + expect(restored?.name).toBe('Restored Novel'); + }); + }); + + describe('_restoreNovelAndChapters', () => { + it('does not replace an unrelated novel when backup IDs collide', async () => { + const testDb = getTestDb(); + await insertTestNovel(testDb, { + path: '/existing/novel', + pluginId: 'existing-plugin', + name: 'Existing Novel', + inLibrary: true, + }); + + const mapping = await _restoreNovelAndChapters( + { + id: 1, + path: '/restored/novel', + pluginId: 'restored-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 10, + novelId: 1, + path: '/restored/chapter-1', + name: 'Chapter 1', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { restoreRunId: 'restore-collision' }, + ); + + expect(mapping.restoredNovelId).not.toBe(1); + expect( + (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, + ).toBe('Existing Novel'); + expect( + (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, + ).toBe(mapping.restoredNovelId); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-1'); + }); + it('reports a completed chapter checkpoint for a batch with no chapters', async () => { + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const mappings = await _restoreNovelsAndChapters( + [ + createBackupNovel(1, '/empty/one'), + createBackupNovel(2, '/empty/two'), + ], + { + includeChapterMappings: false, + onProgress: checkpoint => progress.push(checkpoint), + }, + ); + + expect(mappings).toHaveLength(2); + expect(progress).toEqual([ + { stage: 'novels', completed: 2, total: 2 }, + { stage: 'chapters', completed: 0, total: 0 }, + { stage: 'stats', completed: 0, total: 2 }, + { stage: 'stats', completed: 2, total: 2 }, + ]); + }); + it('restores multiple novels in one batch', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 100, + path: '/bulk/one', + pluginId: 'bulk-plugin', + name: 'Bulk One', + chapters: [], + }, + { + id: 101, + path: '/bulk/two', + pluginId: 'bulk-plugin', + name: 'Bulk Two', + chapters: [], + }, + ], + { includeChapterMappings: false }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings.map(mapping => mapping.pluginId)).toEqual([ + 'bulk-plugin', + 'bulk-plugin', + ]); + expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk One' }), + ); + expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk Two' }), + ); + }); + it('restores 101 novels across a full statement and tail', async () => { + const testDb = getTestDb(); + const unrelatedNovelId = await insertTestNovel(testDb, { + path: '/collision/unrelated', + pluginId: 'unrelated-plugin', + name: 'Unrelated Novel', + }); + const matchingNovelId = await insertTestNovel(testDb, { + path: '/bulk/novel-1', + pluginId: 'restore-plugin', + name: 'Before Restore', + }); + const firstNovelChapters = [ + createBackupChapter(51_001, 1, '/bulk/chapter-1-1', { + updatedTime: '2024-02-01T00:00:00.000Z', + }), + createBackupChapter(51_002, 1, '/bulk/chapter-1-2', { + updatedTime: '2024-02-01 01:00:00', + }), + ]; + const backupNovels = Array.from({ length: 101 }, (_, index) => + createBackupNovel( + index + 1, + `/bulk/novel-${index + 1}`, + index === 0 ? firstNovelChapters : [], + ), + ); + + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const mappings = await _restoreNovelsAndChapters(backupNovels, { + restoreRunId: 'restore-101-novels', + onProgress: checkpoint => progress.push(checkpoint), + }); + expect(progress.filter(({ stage }) => stage === 'novels')).toEqual([ + { stage: 'novels', completed: 100, total: 101 }, + { stage: 'novels', completed: 101, total: 101 }, + ]); + expect(progress.filter(({ stage }) => stage === 'chapters')).toEqual([ + { stage: 'chapters', completed: 2, total: 2 }, + ]); + expect(progress.filter(({ stage }) => stage === 'stats')).toEqual([ + { stage: 'stats', completed: 0, total: 101 }, + { stage: 'stats', completed: 101, total: 101 }, + ]); + + expect(mappings.map(mapping => mapping.backupNovelId)).toEqual( + Array.from({ length: 101 }, (_, index) => index + 1), + ); + expect(mappings[0].restoredNovelId).toBe(matchingNovelId); + expect(mappings[0].restoredNovelId).not.toBe(unrelatedNovelId); + const restoredNovels = await testDb.drizzleDb + .select() + .from(novelSchema) + .all(); + const restoredIdsByIdentity = new Map( + restoredNovels.map(novel => [ + `${novel.pluginId}\u0000${novel.path}`, + novel.id, + ]), + ); + expect( + mappings.map(mapping => + restoredIdsByIdentity.get( + `restore-plugin\u0000/bulk/novel-${mapping.backupNovelId}`, + ), + ), + ).toEqual(mappings.map(mapping => mapping.restoredNovelId)); + expect( + await getNovelByPath('/collision/unrelated', 'unrelated-plugin'), + ).toEqual( + expect.objectContaining({ + id: unrelatedNovelId, + name: 'Unrelated Novel', + }), + ); + expect(await getNovelByPath('/bulk/novel-1', 'restore-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + lastUpdatedAt: '2024-02-01 01:00:00', + }), + ); + expect(await getNovelByPath('/bulk/novel-101', 'restore-plugin')).toEqual( + expect.objectContaining({ totalChapters: 0 }), + ); + }); + it('reports each completed 10,000-chapter database chunk', async () => { + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const chapters = Array.from({ length: 10_001 }, (_, index) => + createBackupChapter( + index + 1, + 1, + `/restore/checkpoints/chapter-${index + 1}`, + ), + ); + + const [mapping] = await _restoreNovelsAndChapters( + [createBackupNovel(1, '/restore/checkpoints', chapters)], + { + includeChapterMappings: false, + onProgress: checkpoint => progress.push(checkpoint), + }, + ); + + expect(progress.filter(({ stage }) => stage === 'chapters')).toEqual([ + { stage: 'chapters', completed: 10_000, total: 10_001 }, + { stage: 'chapters', completed: 10_001, total: 10_001 }, + ]); + expect( + await getNovelByPath('/restore/checkpoints', 'restore-plugin'), + ).toEqual(expect.objectContaining({ totalChapters: 10_001 })); + const persistedChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(persistedChapters).toHaveLength(10_001); + }); + it('restores 201 chapters with stable identities and ordered mappings', async () => { + const testDb = getTestDb(); + const restoredNovelId = await insertTestNovel(testDb, { + path: '/restore/large', + pluginId: 'large-plugin', + name: 'Before Restore', + }); + const stableChapterId = await insertTestChapter(testDb, restoredNovelId, { + path: '/restore/large/chapter-0', + name: 'Before Chapter', + readTime: '2023-01-01T00:00:00.000Z', + updatedTime: '2022-01-01T00:00:00.000Z', + }); + const existingOnlyChapterId = await insertTestChapter( + testDb, + restoredNovelId, + { + path: '/restore/large/existing-only', + name: 'Existing Only', + readTime: '2024-01-01T00:00:00.000Z', + updatedTime: '2023-01-01T00:00:00.000Z', + }, + ); + const chapters = Array.from({ length: 201 }, (_, index) => { + const backupChapterId = + index === 1 || index === 2 + ? 7000 + : index === 99 || index === 100 + ? 8000 + : 10_000 + index; + return createBackupChapter( + backupChapterId, + 999, + `/restore/large/chapter-${index}`, + { + unread: index % 2 === 0, + isDownloaded: index % 3 === 0, + ...(index === 0 + ? { + name: 'Restored Chapter', + releaseTime: '2024-03-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 10, + page: '8', + progress: 45, + position: 7, + scanlator: 'Backup Scanlator', + timeSpent: 12, + } + : {}), + ...(index === 1 ? { updatedTime: '2024-02-01 01:00:00' } : {}), + ...(index === 2 ? { name: 'Same Statement Last' } : {}), + ...(index === 100 ? { name: 'Across Boundary Last' } : {}), + }, + ); + }); + const backupNovel = createBackupNovel( + 999, + '/restore/large', + chapters, + 'large-plugin', + ); + const progress: { + stage: 'novels' | 'chapters' | 'stats'; + completed: number; + total: number; + }[] = []; + const options = { + restoreRunId: 'restore-large', + includeChapterMappings: true, + onProgress: (checkpoint: (typeof progress)[number]) => + progress.push(checkpoint), + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, options); + expect(progress).toEqual([ + { stage: 'novels', completed: 1, total: 1 }, + { stage: 'chapters', completed: 201, total: 201 }, + { stage: 'stats', completed: 0, total: 1 }, + { stage: 'stats', completed: 1, total: 1 }, + ]); + expect(firstMapping.restoredNovelId).toBe(restoredNovelId); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(202); + const chaptersByPath = new Map( + restoredChapters.map(chapter => [chapter.path, chapter]), + ); + expect(chaptersByPath.get('/restore/large/chapter-0')).toEqual( + expect.objectContaining({ + id: stableChapterId, + name: 'Restored Chapter', + releaseTime: '2024-03-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: true, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 10, + page: '8', + progress: 45, + position: 7, + scanlator: 'Backup Scanlator', + timeSpent: 12, + }), + ); + expect(chaptersByPath.get('/restore/large/existing-only')).toEqual( + expect.objectContaining({ + id: existingOnlyChapterId, + name: 'Existing Only', + }), + ); + const mappingRows = await getRestoreChapterMappings( + 'restore-large', + 999, + [10_000, 7000, 8000], + ); + expect(mappingRows).toHaveLength(3); + const mappingsByBackupId = new Map( + mappingRows.map(row => [row.backupChapterId, row]), + ); + expect(mappingsByBackupId.get(10_000)?.restoredChapterId).toBe( + stableChapterId, + ); + expect(mappingsByBackupId.get(7000)?.restoredChapterId).toBe( + chaptersByPath.get('/restore/large/chapter-2')?.id, + ); + expect(mappingsByBackupId.get(8000)?.restoredChapterId).toBe( + chaptersByPath.get('/restore/large/chapter-100')?.id, + ); + expect(await getNovelByPath('/restore/large', 'large-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 202, + chaptersDownloaded: 67, + chaptersUnread: 102, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-02-01 01:00:00', + }), + ); + + await _restoreNovelAndChapters(backupNovel, options); + + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(chaptersAfterReplay).toHaveLength(202); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + restoredChapters.map(chapter => chapter.id).sort((a, b) => a - b), + ); + expect( + await getRestoreChapterMappings('restore-large', 999, [7000, 8000]), + ).toEqual( + expect.arrayContaining([ + { + backupChapterId: 7000, + restoredChapterId: chaptersByPath.get('/restore/large/chapter-2') + ?.id, + }, + { + backupChapterId: 8000, + restoredChapterId: chaptersByPath.get('/restore/large/chapter-100') + ?.id, + }, + ]), + ); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); + }); + it('preserves mappings and aggregate stats for multiple novels', async () => { + await _restoreNovelsAndChapters( + [ + { + id: 200, + path: '/bulk/mapped-one', + pluginId: 'bulk-plugin', + name: 'Mapped One', + chapters: [ + { + id: 2001, + novelId: 200, + path: '/bulk/chapter-one', + name: 'Chapter One', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + { + id: 2002, + novelId: 200, + path: '/bulk/chapter-two', + name: 'Chapter Two', + releaseTime: null, + readTime: '2025-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: false, + updatedTime: '2023-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + progress: null, + position: 1, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 201, + path: '/bulk/mapped-two', + pluginId: 'bulk-plugin', + name: 'Mapped Two', + chapters: [ + { + id: 2011, + novelId: 201, + path: '/bulk/chapter-three', + name: 'Chapter Three', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ], + { includeChapterMappings: true, restoreRunId: 'restore-mappings' }, + ); + + const stagedMappingsOne = await getRestoreChapterMappings( + 'restore-mappings', + 200, + [2001, 2002], + ); + const stagedMappingsTwo = await getRestoreChapterMappings( + 'restore-mappings', + 201, + [2011], + ); + expect( + stagedMappingsOne.map(row => row.backupChapterId).sort((a, b) => a - b), + ).toEqual([2001, 2002]); + expect(stagedMappingsTwo.map(row => row.backupChapterId)).toEqual([2011]); + expect( + new Set( + [...stagedMappingsOne, ...stagedMappingsTwo].map( + row => row.restoredChapterId, + ), + ).size, + ).toBe(3); + expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-01-01T00:00:00.000Z', + }), + ); + expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: null, + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + }); + it('restores chapters without allocating ID mappings when requested', async () => { + const mapping = await _restoreNovelAndChapters( + { + id: 2, + path: '/restored/without-mappings', + pluginId: 'restored-plugin', + name: 'Restored Without Mappings', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 20, + novelId: 2, + path: '/restored/chapter-2', + name: 'Chapter 2', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 2, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + const restoredChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + expect( + await getRestoreChapterMappings('without-mappings', 2, [20]), + ).toEqual([]); + }); + it('requires a run ID when chapter mappings are enabled', async () => { + const backupNovel = createBackupNovel(1, '/restore/missing-run', [ + createBackupChapter(10, 1, '/restore/missing-run/chapter'), + ]); + + await expect(_restoreNovelAndChapters(backupNovel)).rejects.toThrow( + 'Restore run ID is required for chapter mappings', + ); + expect( + await getNovelByPath('/restore/missing-run', 'restore-plugin'), + ).toBeUndefined(); + }); + it('falls back after a failed chapter batch and restores triggers', async () => { + const testDb = getTestDb(); + const restoredNovelId = await insertTestNovel(testDb, { + path: '/restore/fallback', + pluginId: 'fallback-plugin', + name: 'Before Restore', + }); + const existingChapterId = await insertTestChapter( + testDb, + restoredNovelId, + { + path: '/restore/fallback/existing', + name: 'Before Existing', + }, + ); + const backupNovel = createBackupNovel( + 40, + '/restore/fallback', + [ + createBackupChapter(4001, 40, '/restore/fallback/existing', { + name: 'Restored Existing', + }), + createBackupChapter(4002, 40, '/restore/fallback/new', { + name: 'New Chapter', + }), + ], + 'fallback-plugin', + ); + const executeBatch = testDb.dbManager.executeBatch.bind(testDb.dbManager); + let batchFailureInjected = false; + let triggersRestoredAfterRollback = false; + const executeBatchSpy = jest + .spyOn(testDb.dbManager, 'executeBatch') + .mockImplementation(async commands => { + if ( + !batchFailureInjected && + commands.some(([sql]) => + sql.includes('DROP TRIGGER IF EXISTS update_novel_stats'), + ) + ) { + batchFailureInjected = true; + const failedCommands = commands.slice(); + failedCommands.splice(3, 0, [ + 'INSERT INTO MissingRestoreTable VALUES (1)', + ] as SQLBatchTuple); + try { + return await executeBatch(failedCommands); + } catch (error) { + const triggerRows = testDb.sqlite.executeSync( + `SELECT name FROM sqlite_master + WHERE type = 'trigger' + AND name IN ( + 'update_novel_stats', + 'update_novel_stats_on_update', + 'update_novel_stats_on_delete' + )`, + ).rows; + triggersRestoredAfterRollback = triggerRows.length === 3; + throw error; + } + } + return executeBatch(commands); + }); + + try { + await _restoreNovelAndChapters(backupNovel, { + restoreRunId: 'restore-fallback', + }); + } finally { + executeBatchSpy.mockRestore(); + } + + expect(batchFailureInjected).toBe(true); + expect(triggersRestoredAfterRollback).toBe(true); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/fallback/existing', + )?.id, + ).toBe(existingChapterId); + expect( + await getRestoreChapterMappings('restore-fallback', 40, [4001, 4002]), + ).toHaveLength(2); + + await insertTestChapter(testDb, restoredNovelId, { + path: '/restore/fallback/trigger-check', + }); + expect( + await getNovelByPath('/restore/fallback', 'fallback-plugin'), + ).toEqual(expect.objectContaining({ totalChapters: 3 })); + }); + it('merges restored identities without replacing existing rows', async () => { + const testDb = getTestDb(); + const existingNovelId = await insertTestNovel(testDb, { + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Before Restore', + cover: 'before-cover', + summary: 'before-summary', + author: 'Before Author', + artist: 'Before Artist', + status: 'Paused', + genres: 'before', + inLibrary: false, + isLocal: false, + totalPages: 1, + }); + const conflictingChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/chapter', + name: 'Before Chapter', + bookmark: false, + unread: true, + readTime: '2023-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2022-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + position: 0, + progress: 1, + timeSpent: 2, + }, + ); + const unrelatedChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/unrelated', + name: 'Unrelated Chapter', + bookmark: false, + unread: true, + readTime: '2024-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + position: 1, + progress: 0, + timeSpent: 3, + }, + ); + expect(existingNovelId).not.toBe(900); + expect(conflictingChapterId).not.toBe(9001); + + const restoreRunId = 'restore-merge'; + const backupNovel = { + id: 900, + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Restored Novel', + cover: 'file:///mock/novel/storage/cache-key?size=large', + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: true, + isLocal: true, + totalPages: 123, + chapters: [ + { + id: 9001, + novelId: 900, + path: '/restore/merge/chapter', + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }, + ], + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(firstMapping).toEqual({ + pluginId: 'merge-plugin', + backupNovelId: 900, + restoredNovelId: existingNovelId, + }); + + const restoredNovel = await getNovelByPath( + '/restore/merge', + 'merge-plugin', + ); + expect(restoredNovel).toEqual( + expect.objectContaining({ + id: existingNovelId, + name: 'Restored Novel', + cover: `file:///mock/novel/storage/merge-plugin/${existingNovelId}/cover.png?size=large`, + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: 1, + isLocal: 1, + totalPages: 123, + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/chapter', + ), + ).toEqual( + expect.objectContaining({ + id: conflictingChapterId, + novelId: existingNovelId, + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }), + ); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/unrelated', + ), + ).toEqual( + expect.objectContaining({ + id: unrelatedChapterId, + name: 'Unrelated Chapter', + }), + ); + + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([ + { + backupChapterId: 9001, + restoredChapterId: conflictingChapterId, + }, + ]); + + const secondMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(secondMapping.restoredNovelId).toBe(existingNovelId); + const novelsAfterReplay = await testDb.drizzleDb + .select() + .from(novelSchema) + .where( + and( + eq(novelSchema.path, '/restore/merge'), + eq(novelSchema.pluginId, 'merge-plugin'), + ), + ) + .all(); + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(novelsAfterReplay).toHaveLength(1); + expect(chaptersAfterReplay).toHaveLength(2); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + [conflictingChapterId, unrelatedChapterId].sort((a, b) => a - b), + ); + + await clearRestoreChapterMappings(restoreRunId); + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([]); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); + }); + }); +}); diff --git a/src/database/queries/__tests__/index.ts b/src/database/queries/__tests__/index.ts index fe7903cab7..2b9fec59f9 100644 --- a/src/database/queries/__tests__/index.ts +++ b/src/database/queries/__tests__/index.ts @@ -122,12 +122,18 @@ export const testModules = { 'removeNovelsFromLibrary', 'getCachedNovels', 'deleteCachedNovels', - 'restoreLibrary', 'updateNovelInfo', 'pickCustomNovelCover', 'updateNovelCategoryById', 'updateNovelCategories', + ], + }, + NovelRestoreQueries: { + description: 'Novel restore operations', + functions: [ + 'restoreLibrary', '_restoreNovelAndChapters', + '_restoreNovelsAndChapters', ], }, RepositoryQueries: { diff --git a/src/database/queries/__tests__/testData.ts b/src/database/queries/__tests__/testData.ts index 834b2f5d7e..cd2761495d 100644 --- a/src/database/queries/__tests__/testData.ts +++ b/src/database/queries/__tests__/testData.ts @@ -22,6 +22,7 @@ import { export function clearAllTables(testDb: TestDb) { const { sqlite } = testDb; sqlite.executeSync('DELETE FROM NovelCategory'); + sqlite.executeSync('DELETE FROM RestoreChapterMapping'); sqlite.executeSync('DELETE FROM Chapter'); sqlite.executeSync('DELETE FROM Novel'); sqlite.executeSync('DELETE FROM Repository'); diff --git a/src/database/queries/__tests__/testDb.ts b/src/database/queries/__tests__/testDb.ts index 0232c41bdd..dcc43981fd 100644 --- a/src/database/queries/__tests__/testDb.ts +++ b/src/database/queries/__tests__/testDb.ts @@ -30,7 +30,7 @@ const MIGRATION_STATEMENTS = [ `CREATE INDEX IF NOT EXISTS category_sort_idx ON Category (sort)`, `CREATE TABLE IF NOT EXISTS Chapter ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, - novelId integer NOT NULL, + novelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, path text NOT NULL, name text NOT NULL, releaseTime text, @@ -70,6 +70,17 @@ const MIGRATION_STATEMENTS = [ )`, `CREATE UNIQUE INDEX IF NOT EXISTS novel_path_plugin_unique ON Novel (path, pluginId)`, `CREATE INDEX IF NOT EXISTS NovelIndex ON Novel (pluginId, path, id, inLibrary)`, + `CREATE TABLE IF NOT EXISTS RestoreChapterMapping ( + restoreRunId text NOT NULL, + backupNovelId integer NOT NULL, + backupChapterId integer NOT NULL, + restoredNovelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, + restoredChapterId integer NOT NULL REFERENCES Chapter(id) ON DELETE CASCADE +)`, + `CREATE UNIQUE INDEX IF NOT EXISTS restore_chapter_mapping_unique + ON RestoreChapterMapping (restoreRunId, backupNovelId, backupChapterId)`, + `CREATE INDEX IF NOT EXISTS restore_chapter_mapping_novel_index + ON RestoreChapterMapping (restoreRunId, backupNovelId)`, `CREATE TABLE IF NOT EXISTS NovelCategory ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, novelId integer NOT NULL, diff --git a/src/database/schema/chapter.ts b/src/database/schema/chapter.ts index 08354a1282..c4720deb9f 100644 --- a/src/database/schema/chapter.ts +++ b/src/database/schema/chapter.ts @@ -6,12 +6,15 @@ import { text, uniqueIndex, } from 'drizzle-orm/sqlite-core'; +import { novel } from './novel'; export const chapter = sqliteTable( 'Chapter', { id: integer('id').primaryKey({ autoIncrement: true }), - novelId: integer('novelId').notNull(), + novelId: integer('novelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), path: text('path').notNull(), name: text('name').notNull(), releaseTime: text('releaseTime'), diff --git a/src/database/schema/index.ts b/src/database/schema/index.ts index 744c84cd73..2fa0c4edea 100644 --- a/src/database/schema/index.ts +++ b/src/database/schema/index.ts @@ -3,6 +3,7 @@ import { novel } from './novel'; import { chapter } from './chapter'; import { novelCategory } from './novelCategory'; import { repository } from './repository'; +import { restoreChapterMapping } from './restoreChapterMapping'; export { category as categorySchema, @@ -25,6 +26,11 @@ export { type RepositoryRow, type RepositoryInsert, } from './repository'; +export { + restoreChapterMapping as restoreChapterMappingSchema, + type RestoreChapterMappingRow, + type RestoreChapterMappingInsert, +} from './restoreChapterMapping'; /** * Unified schema object containing all database tables @@ -36,6 +42,7 @@ export const schema = { chapter, novelCategory, repository, + restoreChapterMapping, } as const; export type Schema = typeof schema; diff --git a/src/database/schema/restoreChapterMapping.ts b/src/database/schema/restoreChapterMapping.ts new file mode 100644 index 0000000000..c47f1f68f0 --- /dev/null +++ b/src/database/schema/restoreChapterMapping.ts @@ -0,0 +1,40 @@ +import { + index, + integer, + sqliteTable, + text, + uniqueIndex, +} from 'drizzle-orm/sqlite-core'; +import { chapter } from './chapter'; +import { novel } from './novel'; + +export const restoreChapterMapping = sqliteTable( + 'RestoreChapterMapping', + { + restoreRunId: text('restoreRunId').notNull(), + backupNovelId: integer('backupNovelId').notNull(), + backupChapterId: integer('backupChapterId').notNull(), + restoredNovelId: integer('restoredNovelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), + restoredChapterId: integer('restoredChapterId') + .notNull() + .references(() => chapter.id, { onDelete: 'cascade' }), + }, + table => [ + uniqueIndex('restore_chapter_mapping_unique').on( + table.restoreRunId, + table.backupNovelId, + table.backupChapterId, + ), + index('restore_chapter_mapping_novel_index').on( + table.restoreRunId, + table.backupNovelId, + ), + ], +); + +export type RestoreChapterMappingRow = + typeof restoreChapterMapping.$inferSelect; +export type RestoreChapterMappingInsert = + typeof restoreChapterMapping.$inferInsert; diff --git a/src/database/types/index.ts b/src/database/types/index.ts index 349bfaf0bb..5c260c7601 100644 --- a/src/database/types/index.ts +++ b/src/database/types/index.ts @@ -125,10 +125,6 @@ export interface RestoredNovelMapping { pluginId: string; backupNovelId: number; restoredNovelId: number; - chapters: { - backupChapterId: number; - restoredChapterId: number; - }[]; } export interface BackupCategory extends Category { diff --git a/src/i18n/languages/en/strings.json b/src/i18n/languages/en/strings.json index f2c1401d18..ea003d16f2 100644 --- a/src/i18n/languages/en/strings.json +++ b/src/i18n/languages/en/strings.json @@ -122,9 +122,12 @@ "categoryFileWriteFailed": "Failed to write category file: %{error}", "createBackup": "Create backup", "createBackupDesc": "Can be used to restore current library", + "copyingBackup": "Copying backup", "downloadingData": "Downloading Data", "downloadingDownloadedFiles": "Downloading Downloaded Files", + "extractingBackup": "Extracting backup", "failed": "Backup failed", + "finalizingRestore": "Finishing restore", "novelsRestored": "Restored %{count} novels", "novelsRestoredWithErrors": "Restored %{count} novels (%{failedCount} failed)", "novelsBackupFailedSummary": { @@ -141,8 +144,12 @@ "novelDirectoryReadFailed": "Failed to read novel directory: %{error}", "restoringCategories": "Restoring Categories", "restoringCategoriesProgress": "Restoring categories (%{current}/%{total})", + "validatingNovels": "Validating Novels", + "validatingNovelsProgress": "Validating novels (%{current}/%{total})", "restoringNovels": "Restoring Novels", "restoringNovelsProgress": "Restoring novels (%{current}/%{total})", + "restoringNovelsCount": "Restoring novels (%{current})", + "restoringPlugins": "Restoring plugins", "restoringSettings": "Restoring Settings", "missingPluginsAfterRestore": "Missing plugins: %{plugins}", "sectionsRestoreFailedSummary": { @@ -194,6 +201,7 @@ "remoteBackup": "Remote Backup", "restoreBackup": "Restore backup", "restoreBackupDesc": "Restore library from backup file", + "restoreMayBePartial": "Changes already restored will remain if you pause, cancel, or the restore fails.", "restorinBackup": "Restoring backup", "restoringData": "Restoring Data", "restoringSelectedFiles": "Restoring selected files", diff --git a/src/i18n/types/index.ts b/src/i18n/types/index.ts index 924f4b15b0..287df5dc67 100644 --- a/src/i18n/types/index.ts +++ b/src/i18n/types/index.ts @@ -109,9 +109,12 @@ export interface StringMap { 'backupScreen.categoryFileWriteFailed': 'string'; 'backupScreen.createBackup': 'string'; 'backupScreen.createBackupDesc': 'string'; + 'backupScreen.copyingBackup': 'string'; 'backupScreen.downloadingData': 'string'; 'backupScreen.downloadingDownloadedFiles': 'string'; + 'backupScreen.extractingBackup': 'string'; 'backupScreen.failed': 'string'; + 'backupScreen.finalizingRestore': 'string'; 'backupScreen.novelsRestored': 'string'; 'backupScreen.novelsRestoredWithErrors': 'string'; 'backupScreen.novelsBackupFailedSummary.one': 'string'; @@ -124,8 +127,12 @@ export interface StringMap { 'backupScreen.novelDirectoryReadFailed': 'string'; 'backupScreen.restoringCategories': 'string'; 'backupScreen.restoringCategoriesProgress': 'string'; + 'backupScreen.validatingNovels': 'string'; + 'backupScreen.validatingNovelsProgress': 'string'; 'backupScreen.restoringNovels': 'string'; 'backupScreen.restoringNovelsProgress': 'string'; + 'backupScreen.restoringNovelsCount': 'string'; + 'backupScreen.restoringPlugins': 'string'; 'backupScreen.restoringSettings': 'string'; 'backupScreen.missingPluginsAfterRestore': 'string'; 'backupScreen.sectionsRestoreFailedSummary.one': 'string'; @@ -167,6 +174,7 @@ export interface StringMap { 'backupScreen.remoteBackup': 'string'; 'backupScreen.restoreBackup': 'string'; 'backupScreen.restoreBackupDesc': 'string'; + 'backupScreen.restoreMayBePartial': 'string'; 'backupScreen.restorinBackup': 'string'; 'backupScreen.restoringData': 'string'; 'backupScreen.restoringSelectedFiles': 'string'; diff --git a/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx b/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx index 64613a7b1a..a49cbef933 100644 --- a/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx +++ b/src/screens/settings/SettingsBackupScreen/Components/GoogleDriveModal.tsx @@ -10,7 +10,10 @@ import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; import { exists, getBackups, makeDir } from '@api/drive'; import { DriveFile } from '@api/drive/types'; -import { backgroundTasks } from '@services/backgroundTasks'; +import { + backgroundTasks, + type BackgroundTask, +} from '@services/backgroundTasks'; import { useAppSettings } from '@hooks/persisted'; import { formatDate } from '@utils/dateFormat'; import { @@ -20,6 +23,11 @@ import { } from '@services/backup/options'; import { BackupOptionsList } from './BackupOptions'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + enum BackupModal { UNAUTHORIZED, AUTHORIZED, @@ -174,10 +182,12 @@ function RestoreBackup({ theme, setBackupModal, closeModal, + onRestoreSelected, }: { theme: ThemeColors; setBackupModal: (backupModal: BackupModal) => void; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; }) { const [backupList, setBackupList] = useState([]); const { dateFormat = 'default', relativeTimestamps = true } = @@ -212,10 +222,7 @@ function RestoreBackup({ style={styles.btnOutline} onPress={() => { closeModal(); - backgroundTasks.enqueue({ - name: 'DRIVE_RESTORE', - data: item, - }); + onRestoreSelected({ name: 'DRIVE_RESTORE', data: item }); }} > @@ -245,12 +252,14 @@ interface GoogleDriveModalProps { visible: boolean; theme: ThemeColors; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; } export default function GoogleDriveModal({ visible, theme, closeModal, + onRestoreSelected, }: GoogleDriveModalProps) { const [backupModal, setBackupModal] = useState(BackupModal.UNAUTHORIZED); const [user, setUser] = useState(null); @@ -303,6 +312,7 @@ export default function GoogleDriveModal({ theme={theme} setBackupModal={setBackupModal} closeModal={closeModal} + onRestoreSelected={onRestoreSelected} /> ); } diff --git a/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx b/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx new file mode 100644 index 0000000000..2709e28548 --- /dev/null +++ b/src/screens/settings/SettingsBackupScreen/Components/RestoreConfirmationDialog.tsx @@ -0,0 +1,33 @@ +import { Dialog } from '@components'; +import { getString } from '@i18n/translations'; + +interface RestoreConfirmationDialogProps { + visible: boolean; + onCancel: () => void; + onRestore: () => void; +} + +const RestoreConfirmationDialog = ({ + visible, + onCancel, + onRestore, +}: RestoreConfirmationDialogProps) => ( + + + {getString('backupScreen.restoreBackup')} + + {getString('backupScreen.restoreMayBePartial')} + + + + + {getString('common.cancel')} + + + {getString('common.restore')} + + + +); + +export default RestoreConfirmationDialog; diff --git a/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx b/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx index d3ae2a8a25..bde005063c 100644 --- a/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx +++ b/src/screens/settings/SettingsBackupScreen/Components/SelfHostModal.tsx @@ -1,7 +1,10 @@ import { list } from '@api/remote'; import { Button, Dialog, EmptyView } from '@components'; import { useSelfHost } from '@hooks/persisted/useSelfHost'; -import { backgroundTasks } from '@services/backgroundTasks'; +import { + backgroundTasks, + type BackgroundTask, +} from '@services/backgroundTasks'; import { getString } from '@i18n/translations'; import { ThemeColors } from '@theme/types'; import { fetchTimeout } from '@utils/fetch/fetch'; @@ -16,6 +19,11 @@ import { } from '@services/backup/options'; import { BackupOptionsList } from './BackupOptions'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + enum BackupModal { SET_HOST, CONNECTED, @@ -27,6 +35,7 @@ interface SelfHostModalProps { visible: boolean; theme: ThemeColors; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; } function CreateBackup({ @@ -93,11 +102,13 @@ function RestoreBackup({ theme, setBackupModal, closeModal, + onRestoreSelected, }: { host: string; theme: ThemeColors; setBackupModal: (backupModal: BackupModal) => void; closeModal: () => void; + onRestoreSelected: (task: RestoreTask) => void; }) { const [backupList, setBackupList] = useState([]); useEffect(() => { @@ -128,7 +139,7 @@ function RestoreBackup({ style={styles.btnOutline} onPress={() => { closeModal(); - backgroundTasks.enqueue({ + onRestoreSelected({ name: 'SELF_HOST_RESTORE', data: { host, @@ -249,6 +260,7 @@ export default function SelfHostModal({ visible, theme, closeModal, + onRestoreSelected, }: SelfHostModalProps) { const [backupModal, setBackupModal] = useState(BackupModal.SET_HOST); const { host, setHost } = useSelfHost(); @@ -282,6 +294,7 @@ export default function SelfHostModal({ closeModal={closeModal} setBackupModal={setBackupModal} theme={theme} + onRestoreSelected={onRestoreSelected} /> ); } diff --git a/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx b/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx new file mode 100644 index 0000000000..f4150857ab --- /dev/null +++ b/src/screens/settings/SettingsBackupScreen/__tests__/SettingsBackupScreen.test.tsx @@ -0,0 +1,428 @@ +import { fireEvent, render, screen } from '@test-utils'; +import { type ElementType, type ReactElement, type ReactNode } from 'react'; +import { backgroundTasks } from '@services/backgroundTasks'; +import SettingsBackupScreen from '../index'; + +type MockReactModule = { + Fragment: ElementType; + createElement: ( + type: ElementType, + props?: Record | null, + ...children: ReactNode[] + ) => ReactElement; + useState(initialState: T): [T, (value: T | ((previous: T) => T)) => void]; +}; + +type MockReactNativeModule = { + Pressable: ElementType; + Text: ElementType; + TextInput: ElementType; + View: ElementType; +}; + +type MockChildrenProps = { children?: ReactNode }; +type MockButtonProps = MockChildrenProps & { + disabled?: boolean; + onPress?: () => void; + title?: string; +}; +type MockDialogRootProps = MockChildrenProps & { visible: boolean }; +type MockListItemProps = { onPress?: () => void; title: string }; +type MockListInfoItemProps = { title: string }; +type MockFlatListProps = { + data: readonly Item[]; + renderItem: (info: { item: Item; index: number }) => ReactNode; +}; + +jest.mock('@screens/novel/NovelContext', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const NovelContextProvider = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { NovelContextProvider }; +}); + +const partialRestoreWarning = + 'Changes already restored will remain if you pause, cancel, or the restore fails.'; + +jest.mock('@hooks/persisted', () => ({ + useAppSettings() { + 'use no memo'; + return { + automaticBackupIntervalHours: 0, + automaticBackupDirectoryName: undefined, + automaticBackupDirectoryUri: undefined, + dateFormat: 'default', + lastAutomaticBackupAt: undefined, + relativeTimestamps: true, + setAppSettings: jest.fn(), + }; + }, + useTheme() { + 'use no memo'; + return { + background: '#111111', + error: '#cc0000', + onSurface: '#222222', + onSurfaceDisabled: '#666666', + onSurfaceVariant: '#444444', + outline: '#555555', + primary: '#0000cc', + rippleColor: '#dddddd', + secondary: '#777777', + }; + }, +})); + +jest.mock('@hooks/persisted/useTheme', () => { + 'use no memo'; + return { + ThemeProvider: ({ children }: MockChildrenProps) => { + 'use no memo'; + return children; + }, + useTheme() { + 'use no memo'; + return { + background: '#111111', + error: '#cc0000', + onSurface: '#222222', + onSurfaceDisabled: '#666666', + onSurfaceVariant: '#444444', + outline: '#555555', + primary: '#0000cc', + rippleColor: '#dddddd', + secondary: '#777777', + }; + }, + }; +}); + +jest.mock('@hooks', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const useBoolean = () => { + 'use no memo'; + const [value, setValue] = ReactRuntime.useState(false); + return { + value, + setFalse() { + setValue(false); + }, + setTrue() { + setValue(true); + }, + }; + }; + + return { useBoolean }; +}); + +jest.mock('@hooks/persisted/useSelfHost', () => ({ + useSelfHost() { + 'use no memo'; + return { + host: 'https://backup.example', + setHost: jest.fn(), + }; + }, +})); + +jest.mock('@services/backgroundTasks', () => ({ + backgroundTasks: { enqueue: jest.fn() }, + configureAutomaticBackups: jest.fn().mockResolvedValue(undefined), +})); + +jest.mock('@modules/native-file', () => ({ + __esModule: true, + default: { + ExternalDirectoryPath: '/backups', + createDocument: jest.fn(), + pickDirectory: jest.fn(), + pickDocument: jest.fn().mockResolvedValue('file://local-backup.zip'), + }, +})); + +jest.mock('@api/drive', () => ({ + exists: jest.fn().mockResolvedValue({ id: 'drive-root' }), + getBackups: jest.fn().mockResolvedValue([ + { + id: 'drive-archive-id', + name: 'drive-archive.backup', + createdTime: '2026-09-24T00:00:00.000Z', + }, + ]), + makeDir: jest.fn(), +})); + +jest.mock('@api/remote', () => ({ + list: jest.fn().mockResolvedValue(['selfhost-archive.backup']), +})); + +jest.mock('@utils/fetch/fetch', () => ({ + fetchTimeout: jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ name: 'LNReader' }), + }), +})); + +jest.mock('@utils/dateFormat', () => ({ + formatDate: jest.fn(() => 'Sep 24'), +})); + +jest.mock('@utils/showToast', () => ({ showToast: jest.fn() })); + +jest.mock('@i18n/translations', () => { + 'use no memo'; + return { + getString(key: string) { + return key === 'backupScreen.restoreMayBePartial' + ? 'Changes already restored will remain if you pause, cancel, or the restore fails.' + : key; + }, + }; +}); + +jest.mock('@react-native-google-signin/google-signin', () => ({ + GoogleSignin: { + configure: jest.fn(), + getCurrentUser: jest.fn(() => ({ user: { email: 'reader@example.com' } })), + hasPreviousSignIn: jest.fn(() => true), + }, +})); + +jest.mock('expo-clipboard', () => ({ + setStringAsync: jest.fn().mockResolvedValue(true), +})); + +jest.mock('@components', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { Pressable, Text, View } = + jest.requireActual('react-native'); + + const ViewContent = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(View, null, children); + }; + const TextContent = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(Text, null, children); + }; + const Button = ({ children, disabled, onPress, title }: MockButtonProps) => { + 'use no memo'; + return ReactRuntime.createElement( + Pressable, + { accessibilityRole: 'button', disabled, onPress }, + title + ? ReactRuntime.createElement(Text, null, title) + : ReactRuntime.createElement(Text, null, children), + ); + }; + const Dialog = { + Root: ({ children, visible }: MockDialogRootProps) => { + 'use no memo'; + return visible ? ReactRuntime.createElement(View, null, children) : null; + }, + Header: ViewContent, + Title: TextContent, + Description: TextContent, + Content: ViewContent, + Actions: ViewContent, + Action: Button, + ScrollArea: ViewContent, + }; + const List = { + Section: ViewContent, + SubHeader: TextContent, + Item: ({ onPress, title }: MockListItemProps) => { + 'use no memo'; + return ReactRuntime.createElement( + Pressable, + { accessibilityRole: 'button', onPress }, + ReactRuntime.createElement(Text, null, title), + ); + }, + InfoItem: ({ title }: MockListInfoItemProps) => { + 'use no memo'; + return ReactRuntime.createElement(Text, null, title); + }, + }; + + return { + Appbar: () => { + 'use no memo'; + return null; + }, + Button, + Dialog, + EmptyView: () => { + 'use no memo'; + return null; + }, + List, + SafeAreaView: ViewContent, + }; +}); +jest.mock('react-native-gesture-handler', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { Pressable, View } = + jest.requireActual('react-native'); + const FlatList = ({ data, renderItem }: MockFlatListProps) => { + 'use no memo'; + return ReactRuntime.createElement( + View, + null, + data.map((item, index) => + ReactRuntime.createElement( + ReactRuntime.Fragment, + { key: String(index) }, + renderItem({ item, index }), + ), + ), + ); + }; + + return { + FlatList, + GestureHandlerRootView: View, + ScrollView: View, + TouchableOpacity: Pressable, + }; +}); + +jest.mock('react-native-safe-area-context', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const SafeAreaProvider = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { SafeAreaProvider }; +}); +jest.mock('react-native-paper', () => { + 'use no memo'; + const ReactRuntime = jest.requireActual('react'); + const { TextInput } = + jest.requireActual('react-native'); + const PassThrough = ({ children }: MockChildrenProps) => { + 'use no memo'; + return ReactRuntime.createElement(ReactRuntime.Fragment, null, children); + }; + return { Portal: PassThrough, Provider: PassThrough, TextInput }; +}); +jest.mock('../Components/BackupOptions', () => ({ + BackupOptionsDialog: () => { + 'use no memo'; + return null; + }, + BackupOptionsList: () => { + 'use no memo'; + return null; + }, +})); + +jest.mock('../Components/AutomaticBackupDialog', () => ({ + __esModule: true, + AUTOMATIC_BACKUP_LABELS: { 0: 'backupScreen.automaticBackupFrequency' }, + default: () => { + 'use no memo'; + return null; + }, +})); + +const driveBackup = { + id: 'drive-archive-id', + name: 'drive-archive.backup', + createdTime: '2026-09-24T00:00:00.000Z', +}; + +const renderScreen = () => + render( + , + ); + +const selectLocalBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.restoreBackup')); + await screen.findByText(partialRestoreWarning); +}; +const selectDriveBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.googeDrive')); + fireEvent.press(await screen.findByText('common.restore')); + fireEvent.press(await screen.findByText(/drive-archive/)); + await screen.findByText(partialRestoreWarning); +}; + +const selectSelfHostBackup = async () => { + fireEvent.press(screen.getByText('backupScreen.selfHost')); + fireEvent.press(await screen.findByText('common.ok')); + fireEvent.press(await screen.findByText('common.restore')); + fireEvent.press(await screen.findByText(/selfhost-archive/)); + await screen.findByText(partialRestoreWarning); +}; + +const restoreSelections = [ + { + source: 'local', + select: selectLocalBackup, + task: { + name: 'LOCAL_RESTORE', + data: { sourceUri: 'file://local-backup.zip' }, + }, + }, + { + source: 'Drive', + select: selectDriveBackup, + task: { name: 'DRIVE_RESTORE', data: driveBackup }, + }, + { + source: 'self-host', + select: selectSelfHostBackup, + task: { + name: 'SELF_HOST_RESTORE', + data: { + host: 'https://backup.example', + backupFolder: 'selfhost-archive.backup', + }, + }, + }, +]; + +describe('SettingsBackupScreen restore confirmation', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each(restoreSelections)( + 'waits for Restore before enqueueing the $source backup and shows the partial warning', + async ({ select, task }) => { + renderScreen(); + + await select(); + + expect(screen.getByText(partialRestoreWarning)).toBeTruthy(); + expect(backgroundTasks.enqueue).not.toHaveBeenCalled(); + + fireEvent.press(screen.getByText('common.restore')); + + expect(backgroundTasks.enqueue).toHaveBeenCalledTimes(1); + expect(backgroundTasks.enqueue).toHaveBeenCalledWith(task); + }, + ); + + it('discards a selected restore when the user cancels', async () => { + renderScreen(); + + await selectLocalBackup(); + fireEvent.press(screen.getByText('common.cancel')); + + expect(backgroundTasks.enqueue).not.toHaveBeenCalled(); + expect(screen.queryByText(partialRestoreWarning)).toBeNull(); + expect(screen.queryByText('common.restore')).toBeNull(); + }); +}); diff --git a/src/screens/settings/SettingsBackupScreen/index.tsx b/src/screens/settings/SettingsBackupScreen/index.tsx index 274b453d6b..1a650107ea 100644 --- a/src/screens/settings/SettingsBackupScreen/index.tsx +++ b/src/screens/settings/SettingsBackupScreen/index.tsx @@ -3,11 +3,13 @@ import { Appbar, List, SafeAreaView } from '@components'; import { useBoolean } from '@hooks'; import { BackupSettingsScreenProps } from '@navigators/types'; import GoogleDriveModal from './Components/GoogleDriveModal'; +import RestoreConfirmationDialog from './Components/RestoreConfirmationDialog'; import SelfHostModal from './Components/SelfHostModal'; import { backgroundTasks, configureAutomaticBackups, type AutomaticBackupInterval, + type BackgroundTask, } from '@services/backgroundTasks'; import { ScrollView } from 'react-native-gesture-handler'; import { getString } from '@i18n/translations'; @@ -26,6 +28,11 @@ import AutomaticBackupDialog, { } from './Components/AutomaticBackupDialog'; import { showToast } from '@utils/showToast'; +type RestoreTask = Extract< + BackgroundTask, + { name: 'LOCAL_RESTORE' | 'DRIVE_RESTORE' | 'SELF_HOST_RESTORE' } +>; + const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const theme = useTheme(); const { @@ -38,6 +45,8 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const [backupOptions, setBackupOptions] = useState({ ...DEFAULT_BACKUP_OPTIONS, }); + const [pendingRestoreTask, setPendingRestoreTask] = + useState(null); const { value: backupOptionsVisible, setFalse: closeBackupOptions, @@ -117,7 +126,7 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { const restoreLocalBackup = async () => { try { const sourceUri = await NativeFile.pickDocument('application/zip'); - backgroundTasks.enqueue({ + setPendingRestoreTask({ name: 'LOCAL_RESTORE', data: { sourceUri }, }); @@ -126,6 +135,13 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { } }; + const confirmRestore = () => { + if (pendingRestoreTask) { + backgroundTasks.enqueue(pendingRestoreTask); + setPendingRestoreTask(null); + } + }; + const { value: selfHostModalVisible, setFalse: closeSelfHostModal, @@ -203,11 +219,18 @@ const BackupSettings = ({ navigation }: BackupSettingsScreenProps) => { visible={googleDriveModalVisible} theme={theme} closeModal={closeGoogleDriveModal} + onRestoreSelected={setPendingRestoreTask} /> + setPendingRestoreTask(null)} + onRestore={confirmRestore} /> ({ ? 'Download' : key === 'common.preparing' ? 'Preparing' + : key === 'backupScreen.restoreMayBePartial' + ? 'Changes already restored will remain if you pause, cancel, or the restore fails.' : 'Completed', })); @@ -81,7 +83,7 @@ describe('BackgroundTaskQueue completion notifications', () => { ); }); - it('localizes failure text before handing it to the native notification', async () => { + it('appends the partial-restore warning to restore failure notifications', async () => { jest .mocked(executeBackgroundTask) .mockRejectedValueOnce(new Error('Invalid backup')); @@ -91,7 +93,31 @@ describe('BackgroundTaskQueue completion notifications', () => { ).rejects.toThrow('Invalid backup'); expect(NativeBackgroundTasks.fail).toHaveBeenCalledWith( 'restore-2', - 'Failed: Invalid backup', + 'Failed: Invalid backup\nChanges already restored will remain if you pause, cancel, or the restore fails.', + false, + ); + }); + + it('leaves unrelated task failure notifications unchanged', async () => { + const downloadTask = { + name: 'DOWNLOAD_CHAPTER' as const, + data: { + novelName: 'Example Novel', + novelId: 42, + pluginId: 'source-a', + chapters: [{ chapterId: 1, chapterName: 'Chapter 1' }], + }, + }; + jest + .mocked(executeBackgroundTask) + .mockRejectedValueOnce(new Error('Network unavailable')); + + await expect( + new BackgroundTaskQueue().run('download-1', downloadTask), + ).rejects.toThrow('Network unavailable'); + expect(NativeBackgroundTasks.fail).toHaveBeenCalledWith( + 'download-1', + 'Failed: Network unavailable', false, ); }); diff --git a/src/services/backgroundTasks/executeTask.ts b/src/services/backgroundTasks/executeTask.ts index b2fd3041ad..be5e213467 100644 --- a/src/services/backgroundTasks/executeTask.ts +++ b/src/services/backgroundTasks/executeTask.ts @@ -1,6 +1,9 @@ -import { createDriveBackup, driveRestore } from '../backup/drive'; -import { createBackup, restoreBackup } from '../backup/local'; -import { createSelfHostBackup, selfHostRestore } from '../backup/selfhost'; +import { createDriveBackup } from '../backup/drive'; +import { driveRestore } from '../backup/restore/drive'; +import { createBackup } from '../backup/local'; +import { restoreBackup } from '../backup/restore/local'; +import { createSelfHostBackup } from '../backup/selfhost'; +import { selfHostRestore } from '../backup/restore/selfhost'; import { downloadChapters } from '../download/downloadChapter'; import { exportEpub } from '../epub/export'; import { importEpubBatch } from '../epub/import'; diff --git a/src/services/backup/__tests__/create.test.ts b/src/services/backup/__tests__/create.test.ts new file mode 100644 index 0000000000..4a45774ce0 --- /dev/null +++ b/src/services/backup/__tests__/create.test.ts @@ -0,0 +1,529 @@ +import { getAllNovels } from '@database/queries/NovelQueries'; +import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; +import { + getAllNovelCategories, + getCategoriesFromDb, +} from '@database/queries/CategoryQueries'; +import NativeFile from '@modules/native-file'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { prepareBackupData } from '../create'; +import { decodeNovelBatch, encodeNovelBatch } from '../novelPayload'; +import type { BackupNovel, ChapterInfo } from '@database/types'; +import type { BackupOptions } from '../options'; + +jest.mock('@database/queries/NovelQueries', () => ({ + getAllNovels: jest.fn(), +})); + +jest.mock('@database/queries/ChapterQueries', () => ({ + getAllNovelChaptersForBackup: jest.fn(), +})); + +jest.mock('@database/queries/CategoryQueries', () => ({ + getAllNovelCategories: jest.fn(), + getCategoriesFromDb: jest.fn(), +})); + +jest.mock('@hooks/persisted/useSelfHost', () => ({ + SELF_HOST_BACKUP: 'SELF_HOST_BACKUP', +})); + +jest.mock('@hooks/persisted/migrations/trackerMigration', () => ({ + OLD_TRACKED_NOVEL_PREFIX: 'OLD_TRACKED_NOVEL_PREFIX', +})); + +jest.mock('@hooks/persisted/useUpdates', () => ({ + LAST_UPDATE_TIME: 'LAST_UPDATE_TIME', +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const pluginOnlyOptions: BackupOptions = { + library: false, + settings: false, + plugins: true, + downloadedFiles: false, +}; + +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +describe('selective backup creation', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(NativeFile.exists).mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(getAllNovels).mockResolvedValue([]); + jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); + jest.mocked(getCategoriesFromDb).mockResolvedValue([]); + jest.mocked(getAllNovelCategories).mockResolvedValue([]); + }); + + it('writes the selected sections to the v2 manifest', async () => { + await prepareBackupData('/cache', pluginOnlyOptions); + + expect(NativeFile.writeFile).toHaveBeenCalledTimes(2); + expect(NativeFile.writeFile).toHaveBeenCalledWith( + '/cache/Version.json', + expect.stringContaining( + '"sections":{"library":false,"settings":false,"plugins":true,"downloadedFiles":false}', + ), + ); + expect(getAllNovels).not.toHaveBeenCalled(); + expect(getCategoriesFromDb).not.toHaveBeenCalled(); + expect(NativeFile.writeFile).toHaveBeenCalledWith( + '/cache/Plugins.json', + '[]', + ); + }); + it('stores the total novel count in the v3 manifest', async () => { + const novels = [makeTestNovel(1), makeTestNovel(2), makeTestNovel(3)]; + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + + await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}'), + ).toMatchObject({ + formatVersion: 3, + novelCount: 3, + }); + }); + + it('counts only validated novels in the v3 manifest', async () => { + const novels = [ + makeTestNovel(1), + { ...makeTestNovel(2), name: 2 as unknown as string }, + ]; + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + + const result = await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1]).novelCount, + ).toBe(1); + expect(result.failedNovelCount).toBe(1); + }); + + it('does not count novels from a failed batch write', async () => { + jest + .mocked(getAllNovels) + .mockResolvedValueOnce([makeTestNovel(1), makeTestNovel(2)]); + jest.mocked(NativeFile.writeFile).mockImplementation(async path => { + if (path.includes('/NovelAndChapters/batch-')) { + throw new Error('Novel batch write failed'); + } + }); + + const result = await prepareBackupData( + '/cache', + { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + 3, + ); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}') + .novelCount, + ).toBe(0); + expect(result.failedNovelCount).toBe(2); + }); + + it('includes stored covers with library data when downloads are omitted', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + ...makeTestNovel(1), + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ + { + ...makeTestChapter(1), + id: 10, + }, + ]); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', + }); + expect(compactNovel.c).toHaveLength(1); + expect(compactNovel.c[0]).toHaveLength(15); + expect(compactNovel.c[0][0]).toBe(10); + expect(compactNovel.c[0][7]).toBe(false); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + }); + + it('does not duplicate covers when downloaded files are included', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + ...makeTestNovel(1), + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + jest.mocked(NativeFile.copyFile).mockClear(); + jest.mocked(NativeFile.mkdir).mockClear(); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', + }); + expect(NativeFile.copyFile).not.toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + expect(NativeFile.mkdir).not.toHaveBeenCalledWith('/cache/Covers'); + }); + + it('round-trips a fully populated novel through the compact codec', () => { + const novel: BackupNovel & { + chaptersDownloaded: number; + chaptersUnread: number; + totalChapters: number; + lastReadAt: string; + lastUpdatedAt: string; + } = { + id: 7, + name: 'The Compact Novel', + path: '/novels/compact', + pluginId: 'source', + cover: '/covers/compact.png?cache=1', + summary: null, + author: 'Author', + artist: null, + status: 'ongoing', + genres: null, + inLibrary: null, + isLocal: false, + totalPages: 2048, + chaptersDownloaded: 1, + chaptersUnread: 1, + totalChapters: 2, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + chapters: [ + { + id: 701, + novelId: 7, + path: '/novels/compact/1', + name: 'First chapter', + releaseTime: '2024-01-01T00:00:00.000Z', + readTime: '2024-03-01T10:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 1, + page: '4', + position: 2, + progress: 0.75, + scanlator: 'Team A', + timeSpent: 90, + }, + { + id: 702, + novelId: 7, + path: '/novels/compact/2', + name: 'Second chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: null, + }, + ], + }; + + const [compactNovel] = encodeNovelBatch([novel]); + + expect(Object.keys(compactNovel).sort()).toEqual( + [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', + ].sort(), + ); + expect(compactNovel).toMatchObject({ + id: 7, + p: '/novels/compact', + pi: 'source', + n: 'The Compact Novel', + co: '/covers/compact.png?cache=1', + d: 1, + u: 1, + tc: 2, + lr: '2024-03-01T10:20:30.000Z', + lu: '2024-03-02T10:20:30.000Z', + }); + expect(compactNovel.c).toEqual([ + [ + 701, + '/novels/compact/1', + 'First chapter', + '2024-01-01T00:00:00.000Z', + true, + false, + '2024-03-01T10:00:00.000Z', + true, + '2024-02-01T00:00:00.000Z', + 1, + '4', + 2, + 0.75, + 'Team A', + 90, + ], + [ + 702, + '/novels/compact/2', + 'Second chapter', + null, + null, + true, + null, + false, + null, + null, + null, + null, + null, + null, + null, + ], + ]); + expect(Object.keys(compactNovel.c[0])).not.toContain('novelId'); + expect(compactNovel).not.toHaveProperty('chapters'); + expect(decodeNovelBatch([compactNovel])).toEqual([novel]); + }); + + it('writes 101 novels as ordered compact batches with a format marker', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels = Array.from({ length: 101 }, (_, index) => { + const id = index + 1; + return makeTestNovel(id, 'source', [ + makeTestChapter(id, 1), + makeTestChapter(id, 2), + ]); + }); + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + jest.mocked(getAllNovelChaptersForBackup).mockImplementation(async ids => { + const requestedIds = Array.isArray(ids) ? ids : [ids]; + return novels + .flatMap(novel => novel.chapters) + .filter(chapter => requestedIds.includes(chapter.novelId)); + }); + + await prepareBackupData('/cache', options); + + const batchWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.includes('/NovelAndChapters/batch-')); + expect(batchWrites.map(([path]) => path)).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(batchWrites.some(([path]) => /\/\d+\.json$/.test(path))).toBe(false); + + const batches = batchWrites.map(([, content]) => JSON.parse(content)); + expect(batches.map(batch => batch.length)).toEqual([100, 1]); + const records = batches.flat(); + expect(records.map(record => record.id)).toEqual( + novels.map(novel => novel.id), + ); + expect( + records.map(record => + record.c.map((chapter: [number, ...unknown[]]) => chapter[0]), + ), + ).toEqual(novels.map(novel => novel.chapters.map(chapter => chapter.id))); + expect( + records.every( + record => + Array.isArray(record.c) && + record.c.every((chapter: unknown[]) => chapter.length === 15), + ), + ).toBe(true); + + const manifestWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.endsWith('/Version.json')); + expect( + JSON.parse(manifestWrites[manifestWrites.length - 1][1] ?? '{}'), + ).toMatchObject({ + formatVersion: 2, + novelDataFormat: 2, + novelCount: 101, + }); + }); + + it('omits the installed-plugin registry when plugin files are excluded', async () => { + jest + .mocked(MMKVStorage.getAllKeys) + .mockReturnValueOnce(['INSTALL_PLUGINS', 'OTHER_SETTING']); + jest + .mocked(MMKVStorage.getString) + .mockImplementation(key => + key === 'INSTALL_PLUGINS' + ? '[{"id":"source"}]' + : key === 'OTHER_SETTING' + ? 'kept' + : undefined, + ); + + await prepareBackupData('/cache', { + library: false, + settings: true, + plugins: false, + downloadedFiles: false, + }); + + const settingsWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => path.endsWith('/Setting.json')); + expect(JSON.parse(settingsWrite?.[1] ?? '{}')).toEqual({ + OTHER_SETTING: 'kept', + }); + }); +}); diff --git a/src/services/backup/__tests__/fileSections.test.ts b/src/services/backup/__tests__/fileSections.test.ts index f7d5bac972..58058dbc50 100644 --- a/src/services/backup/__tests__/fileSections.test.ts +++ b/src/services/backup/__tests__/fileSections.test.ts @@ -1,70 +1,34 @@ -import NativeFile from '@modules/native-file'; -import { restoreNovelFiles } from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; +import { ZipBackupName } from '../types'; jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', })); +describe('selected backup file sections', () => { + const options = { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }; -describe('restoreNovelFiles', () => { - it('moves downloaded files to remapped novel and chapter IDs', async () => { - jest.mocked(NativeFile.exists).mockResolvedValue(true); - jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); - jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); - jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); - jest.mocked(NativeFile.readDir).mockImplementation(async path => { - if (path === '/staging/source/1') { - return [ - { - name: 'cover.png', - path: '/staging/source/1/cover.png', - isDirectory: false, - }, - { - name: '10', - path: '/staging/source/1/10', - isDirectory: true, - }, - { - name: '20', - path: '/staging/source/1/20', - isDirectory: true, - }, - ]; - } - if (path === '/staging/source/1/10') { - return [ - { - name: 'index.html', - path: '/staging/source/1/10/index.html', - isDirectory: false, - }, - ]; - } - return []; - }); - - await restoreNovelFiles('/staging', [ + it('keeps novel files in the legacy v2 section only', () => { + expect(getSelectedBackupFileSections(options)).toEqual([ { - pluginId: 'source', - backupNovelId: 1, - restoredNovelId: 7, - chapters: [{ backupChapterId: 10, restoredChapterId: 99 }], + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + ]); + expect(getSelectedBackupFileSections(options, 2)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + { + archiveName: ZipBackupName.NOVEL_FILES, + storagePath: '/storage/Novels', }, ]); - - expect(NativeFile.moveFile).toHaveBeenCalledWith( - '/staging/source/1/cover.png', - '/storage/Novels/source/7/cover.png', - ); - expect(NativeFile.moveFile).toHaveBeenCalledWith( - '/staging/source/1/10/index.html', - '/storage/Novels/source/7/99/index.html', - ); - expect(NativeFile.moveFile).not.toHaveBeenCalledWith( - expect.stringContaining('/20/'), - expect.any(String), - ); - expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); }); }); diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index 5601615128..6c85e67bbe 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -1,19 +1,15 @@ import NativeFile from '@modules/native-file'; import NativeZipArchive from '@modules/native-zip-archive'; -import { createBackup, restoreBackup } from '../local'; -import { finalizeRestoredPlugins } from '../restoreResult'; -import { prepareBackupData, restoreData } from '../utils'; +import { createBackup } from '../local'; +import { prepareBackupData } from '../create'; -jest.mock('../utils', () => ({ +jest.mock('../cache', () => ({ CACHE_DIR_PATH: '/cache/BackupData', clearBackupCache: jest.fn(), - prepareBackupData: jest.fn(), - restoreData: jest.fn(), })); -jest.mock('../restoreResult', () => ({ - finalizeRestoredPlugins: jest.fn(), - getRestoreCompletionText: jest.fn(), +jest.mock('../create', () => ({ + prepareBackupData: jest.fn(), })); jest.mock('../backupResult', () => ({ @@ -23,7 +19,6 @@ jest.mock('../backupResult', () => ({ jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', - ROOT_STORAGE: '/storage', })); jest.mock('@utils/sleep', () => ({ @@ -34,14 +29,23 @@ jest.mock('@i18n/translations', () => ({ getString: (key: string) => key, })); -describe('local selective backup', () => { +describe('local backup creation', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(prepareBackupData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + }); + it('creates archives only for selected file sections', async () => { jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, failedSectionCount: 0, }); - jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); await createBackup({ destinationUri: 'content://backup.zip', @@ -53,12 +57,16 @@ describe('local selective backup', () => { }, }); - expect(prepareBackupData).toHaveBeenCalledWith('/cache/BackupData', { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }); + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + 3, + ); expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', '/cache/BackupData/plugins.zip', @@ -67,50 +75,52 @@ describe('local selective backup', () => { '/storage/Novels', expect.any(String), ); - expect(NativeZipArchive.zip).toHaveBeenCalledWith( - '/cache/BackupData', + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [{ path: '/cache/BackupData', prefix: '' }], '/cache/BackupData.zip', ); }); - it('loads restored plugins after their archive is extracted', async () => { - const restoreResult = { - novelCount: 1, + it('adds novel files to the v3 outer archive without a nested archive', async () => { + jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, - categoryCount: 0, - failedCategoryCount: 0, - settingsRestored: true, failedSectionCount: 0, - pluginIds: ['restored'], - novelMappings: [], - manifest: { - appVersion: '2.1.0', - formatVersion: 2 as const, - sections: { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }, - }, - }; - jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); - jest.mocked(NativeFile.exists).mockResolvedValue(true); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); - jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); - jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + }); - await restoreBackup({ sourceUri: 'content://backup.zip' }); + await createBackup({ + destinationUri: 'content://backup.zip', + options: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); - expect(NativeZipArchive.unzip).toHaveBeenCalledWith( - '/cache/BackupData/plugins.zip', + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + 3, + ); + expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', + '/cache/BackupData/plugins.zip', ); - expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); - expect( - jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], - ).toBeGreaterThan( - jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], + expect(NativeZipArchive.zip).not.toHaveBeenCalledWith( + '/storage/Novels', + expect.any(String), + ); + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [ + { path: '/cache/BackupData', prefix: '' }, + { path: '/storage/Novels', prefix: 'NovelFiles' }, + ], + '/cache/BackupData.zip', ); }); }); diff --git a/src/services/backup/__tests__/options.test.ts b/src/services/backup/__tests__/options.test.ts index 33fff6b8bf..edf8fe6cb5 100644 --- a/src/services/backup/__tests__/options.test.ts +++ b/src/services/backup/__tests__/options.test.ts @@ -7,6 +7,10 @@ import { import { getSelectedBackupFileSections } from '../fileSections'; import { ZipBackupName } from '../types'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts deleted file mode 100644 index 7c018e283a..0000000000 --- a/src/services/backup/__tests__/utils.test.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { - _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; -import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; -import { - _restoreCategory, - getAllNovelCategories, - getCategoriesFromDb, -} from '@database/queries/CategoryQueries'; -import NativeFile from '@modules/native-file'; -import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { prepareBackupData, restoreData } from '../utils'; -import type { BackupOptions } from '../options'; - -jest.mock('@database/queries/NovelQueries', () => ({ - _restoreNovelAndChapters: jest.fn(), - getAllNovels: jest.fn(), -})); - -jest.mock('@database/queries/ChapterQueries', () => ({ - getAllNovelChaptersForBackup: jest.fn(), -})); - -jest.mock('@database/queries/CategoryQueries', () => ({ - _restoreCategory: jest.fn(), - getAllNovelCategories: jest.fn(), - getCategoriesFromDb: jest.fn(), -})); - -jest.mock('@hooks/persisted/useSelfHost', () => ({ - SELF_HOST_BACKUP: 'SELF_HOST_BACKUP', -})); - -jest.mock('@hooks/persisted/migrations/trackerMigration', () => ({ - OLD_TRACKED_NOVEL_PREFIX: 'OLD_TRACKED_NOVEL_PREFIX', -})); - -jest.mock('@hooks/persisted/useUpdates', () => ({ - LAST_UPDATE_TIME: 'LAST_UPDATE_TIME', -})); - -jest.mock('@utils/mmkv/mmkv', () => ({ - MMKVStorage: { - getAllKeys: jest.fn(() => []), - getBoolean: jest.fn(), - getString: jest.fn(), - set: jest.fn(), - }, -})); - -jest.mock('@i18n/translations', () => ({ - getString: (key: string) => key, -})); - -jest.mock('@plugins/pluginManager', () => ({ - INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', -})); - -jest.mock('@utils/Storages', () => ({ - NOVEL_STORAGE: '/storage/Novels', - ROOT_STORAGE: '/storage', -})); - -const pluginOnlyOptions: BackupOptions = { - library: false, - settings: false, - plugins: true, - downloadedFiles: false, -}; - -describe('selective backup data', () => { - beforeEach(() => { - jest.mocked(NativeFile.exists).mockResolvedValue(false); - jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); - jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); - jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); - jest.mocked(getAllNovels).mockResolvedValue([]); - jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); - jest.mocked(getCategoriesFromDb).mockResolvedValue([]); - jest.mocked(getAllNovelCategories).mockResolvedValue([]); - jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => ({ - pluginId: novel.pluginId, - backupNovelId: novel.id, - restoredNovelId: novel.id, - chapters: novel.chapters.map(chapter => ({ - backupChapterId: chapter.id, - restoredChapterId: chapter.id, - })), - })); - }); - - it('writes the selected sections to the v2 manifest', async () => { - await prepareBackupData('/cache', pluginOnlyOptions); - - expect(NativeFile.writeFile).toHaveBeenCalledTimes(2); - expect(NativeFile.writeFile).toHaveBeenCalledWith( - '/cache/Version.json', - expect.stringContaining( - '"sections":{"library":false,"settings":false,"plugins":true,"downloadedFiles":false}', - ), - ); - expect(getAllNovels).not.toHaveBeenCalled(); - expect(getCategoriesFromDb).not.toHaveBeenCalled(); - expect(NativeFile.writeFile).toHaveBeenCalledWith( - '/cache/Plugins.json', - '[]', - ); - }); - - it('does not warn about sections intentionally omitted by the manifest', async () => { - jest - .mocked(NativeFile.readFile) - .mockResolvedValueOnce( - JSON.stringify({ - appVersion: '2.1.0', - formatVersion: 2, - sections: pluginOnlyOptions, - }), - ) - .mockResolvedValueOnce('[]'); - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => path.endsWith('/Plugins.json')); - - const result = await restoreData('/cache'); - - expect(result).toMatchObject({ - failedNovelCount: 0, - failedCategoryCount: 0, - failedSectionCount: 0, - settingsRestored: true, - manifest: { - formatVersion: 2, - sections: pluginOnlyOptions, - }, - }); - expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); - expect(_restoreCategory).not.toHaveBeenCalled(); - expect(MMKVStorage.set).toHaveBeenCalledWith('INSTALL_PLUGINS', '[]'); - }); - - it('merges restored plugins with the existing registry', async () => { - const options: BackupOptions = { - library: false, - settings: true, - plugins: true, - downloadedFiles: false, - }; - jest - .mocked(NativeFile.exists) - .mockImplementation( - async path => - path.endsWith('/Version.json') || - path.endsWith('/Setting.json') || - path.endsWith('/Plugins.json'), - ); - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ - appVersion: '2.1.0', - formatVersion: 2, - sections: options, - }); - } - if (path.endsWith('/Setting.json')) { - return JSON.stringify({ - INSTALL_PLUGINS: JSON.stringify([ - { id: 'restored', name: 'Restored' }, - ]), - THEME: 'dark', - }); - } - return JSON.stringify([{ id: 'restored', name: 'Restored' }]); - }); - jest - .mocked(MMKVStorage.getString) - .mockReturnValueOnce( - JSON.stringify([{ id: 'existing', name: 'Existing' }]), - ); - - await restoreData('/cache'); - - expect(MMKVStorage.set).toHaveBeenCalledWith('THEME', 'dark'); - expect(MMKVStorage.set).toHaveBeenCalledWith( - 'INSTALL_PLUGINS', - JSON.stringify([ - { id: 'existing', name: 'Existing' }, - { id: 'restored', name: 'Restored' }, - ]), - ); - }); - - it('merges the plugin registry from legacy settings', async () => { - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => path.endsWith('/Setting.json')); - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ version: '2.0.0' }); - } - return JSON.stringify({ - INSTALL_PLUGINS: JSON.stringify([{ id: 'legacy', name: 'Legacy' }]), - }); - }); - jest - .mocked(MMKVStorage.getString) - .mockReturnValueOnce( - JSON.stringify([{ id: 'existing', name: 'Existing' }]), - ); - - await restoreData('/cache'); - - expect(MMKVStorage.set).toHaveBeenCalledWith( - 'INSTALL_PLUGINS', - JSON.stringify([ - { id: 'existing', name: 'Existing' }, - { id: 'legacy', name: 'Legacy' }, - ]), - ); - }); - - it('includes stored covers with library data when downloads are omitted', async () => { - jest.mocked(getAllNovels).mockResolvedValueOnce([ - { - id: 1, - name: 'Example', - path: '/example', - pluginId: 'source', - cover: 'file:///storage/Novels/source/1/cover.png?123', - }, - ]); - jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ - { - id: 10, - novelId: 1, - path: '/chapter-1', - name: 'Chapter 1', - isDownloaded: true, - }, - ] as Awaited>); - - await prepareBackupData('/cache', { - library: true, - settings: false, - plugins: false, - downloadedFiles: false, - }); - - const novelWrite = jest - .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/1.json')); - expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ - cover: '/Novels/source/1/cover.png?123', - chapters: [{ id: 10, isDownloaded: false }], - }); - expect(NativeFile.copyFile).toHaveBeenCalledWith( - 'file:///storage/Novels/source/1/cover.png', - '/cache/Covers/1', - ); - }); - - it('restores stored covers from library data and preserves missing covers', async () => { - const options: BackupOptions = { - library: true, - settings: false, - plugins: false, - downloadedFiles: false, - }; - jest.mocked(NativeFile.readFile).mockImplementation(async path => { - if (path.endsWith('/Version.json')) { - return JSON.stringify({ - appVersion: '2.1.2', - formatVersion: 2, - sections: options, - }); - } - if (path.endsWith('/1.json')) { - return JSON.stringify({ - id: 1, - name: 'Stored cover', - path: '/stored-cover', - pluginId: 'source', - cover: '/Novels/source/1/cover.png?123', - chapters: [], - }); - } - if (path.endsWith('/2.json')) { - return JSON.stringify({ - id: 2, - name: 'Missing cover', - path: '/missing-cover', - pluginId: 'source', - cover: null, - chapters: [], - }); - } - return '[]'; - }); - jest - .mocked(NativeFile.exists) - .mockImplementation(async path => - [ - '/cache/NovelAndChapters', - '/cache/Covers/1', - '/cache/Category.json', - ].includes(path), - ); - jest.mocked(NativeFile.readDir).mockResolvedValue([ - { - name: '1.json', - path: '/cache/NovelAndChapters/1.json', - isDirectory: false, - }, - { - name: '2.json', - path: '/cache/NovelAndChapters/2.json', - isDirectory: false, - }, - ]); - - await restoreData('/cache'); - - expect(NativeFile.copyFile).toHaveBeenCalledWith( - '/cache/Covers/1', - '/storage/Novels/source/1/cover.png', - ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - cover: 'file:///storage/Novels/source/1/cover.png?123', - }), - ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ id: 2, cover: null }), - ); - }); - - it('omits the installed-plugin registry when plugin files are excluded', async () => { - jest - .mocked(MMKVStorage.getAllKeys) - .mockReturnValueOnce(['INSTALL_PLUGINS', 'OTHER_SETTING']); - jest - .mocked(MMKVStorage.getString) - .mockImplementation(key => - key === 'INSTALL_PLUGINS' - ? '[{"id":"source"}]' - : key === 'OTHER_SETTING' - ? 'kept' - : undefined, - ); - - await prepareBackupData('/cache', { - library: false, - settings: true, - plugins: false, - downloadedFiles: false, - }); - - const settingsWrite = jest - .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/Setting.json')); - expect(JSON.parse(settingsWrite?.[1] ?? '{}')).toEqual({ - OTHER_SETTING: 'kept', - }); - }); - - it('treats backups without a section manifest as legacy full backups', async () => { - jest - .mocked(NativeFile.readFile) - .mockResolvedValueOnce(JSON.stringify({ version: '2.0.0' })); - - const result = await restoreData('/cache'); - - expect(result.manifest).toMatchObject({ - appVersion: '2.0.0', - formatVersion: 1, - sections: { - library: true, - settings: true, - plugins: true, - downloadedFiles: true, - }, - }); - }); -}); diff --git a/src/services/backup/cache.ts b/src/services/backup/cache.ts new file mode 100644 index 0000000000..5f4a9a4543 --- /dev/null +++ b/src/services/backup/cache.ts @@ -0,0 +1,10 @@ +import NativeFile from '@modules/native-file'; + +export const CACHE_DIR_PATH = + NativeFile.ExternalCachesDirectoryPath + '/BackupData'; + +export const clearBackupCache = async (cacheDirPath = CACHE_DIR_PATH) => { + if (await NativeFile.exists(cacheDirPath)) { + await NativeFile.unlink(cacheDirPath); + } +}; diff --git a/src/services/backup/create.ts b/src/services/backup/create.ts new file mode 100644 index 0000000000..d5e1397460 --- /dev/null +++ b/src/services/backup/create.ts @@ -0,0 +1,250 @@ +import { SELF_HOST_BACKUP } from '@hooks/persisted/useSelfHost'; +import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMigration'; +import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { version } from '../../../package.json'; +import { getAllNovels } from '@database/queries/NovelQueries'; +import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; +import { + getAllNovelCategories, + getCategoriesFromDb, +} from '@database/queries/CategoryQueries'; +import type { BackupNovel } from '@database/types'; +import { encodeNovelBatch, validateBackupNovel } from './novelPayload'; +import { BackupEntryName, type BackupManifest } from './types'; +import { BACKGROUND_TASKS_STORE_KEY } from '@services/backgroundTasks/constants'; +import NativeFile from '@modules/native-file'; +import type { BackupResult } from './backupResult'; +import { resolveBackupOptions, type BackupOptions } from './options'; +import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; +import { ROOT_STORAGE } from '@utils/Storages'; +import { clearBackupCache } from './cache'; + +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +const BACKUP_NOVEL_BATCH_SIZE = 100; + +const BACKUP_FILE_CONCURRENCY = 8; + +const backupMMKVData = () => { + const excludeKeys = [ + BACKGROUND_TASKS_STORE_KEY, + OLD_TRACKED_NOVEL_PREFIX, + SELF_HOST_BACKUP, + LAST_UPDATE_TIME, + INSTALLED_PLUGINS_KEY, + ]; + const keys = MMKVStorage.getAllKeys().filter( + key => !excludeKeys.includes(key), + ); + const data: Record = {}; + for (const key of keys) { + let value: number | string | boolean | undefined = + MMKVStorage.getString(key); + if (!value) { + value = MMKVStorage.getBoolean(key); + } + if (key && value) { + data[key] = value; + } + } + return data; +}; + +export const prepareBackupData = async ( + cacheDirPath: string, + requestedOptions?: BackupOptions, + formatVersion: BackupManifest['formatVersion'] = 2, +): Promise => { + const options = resolveBackupOptions(requestedOptions); + const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; + const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; + let failedNovelCount = 0; + let failedSectionCount = 0; + + await clearBackupCache(cacheDirPath); + await NativeFile.mkdir(cacheDirPath); + + const manifest: BackupManifest = { + appVersion: version, + formatVersion, + novelDataFormat: 2, + sections: options, + }; + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + JSON.stringify(manifest), + ); + + // novels + if (options.library) { + await NativeFile.mkdir(novelDirPath); + if (!options.downloadedFiles) { + await NativeFile.mkdir(coversDirPath); + } + let backedUpNovelCount = 0; + const novels = await getAllNovels(); + for ( + let start = 0; + start < novels.length; + start += BACKUP_NOVEL_BATCH_SIZE + ) { + const novelBatch = novels.slice(start, start + BACKUP_NOVEL_BATCH_SIZE); + let chapters; + try { + chapters = await getAllNovelChaptersForBackup( + novelBatch.map(novel => novel.id), + ); + } catch { + failedNovelCount += novelBatch.length; + continue; + } + + const chaptersByNovel = new Map(); + for (const chapter of chapters) { + const novelChapters = chaptersByNovel.get(chapter.novelId); + if (novelChapters) { + novelChapters.push(chapter); + } else { + chaptersByNovel.set(chapter.novelId, [chapter]); + } + } + + const preparedNovels: BackupNovel[] = []; + for ( + let fileStart = 0; + fileStart < novelBatch.length; + fileStart += BACKUP_FILE_CONCURRENCY + ) { + const fileBatch = novelBatch.slice( + fileStart, + fileStart + BACKUP_FILE_CONCURRENCY, + ); + const prepared = await Promise.all( + fileBatch.map(async (novel): Promise => { + try { + const novelChapters = chaptersByNovel.get(novel.id) ?? []; + const backedUpChapters = options.downloadedFiles + ? novelChapters + : novelChapters.map(chapter => ({ + ...chapter, + isDownloaded: false, + })); + let cover = novel.cover; + if (cover?.startsWith(APP_STORAGE_URI)) { + if (options.downloadedFiles) { + cover = cover.replace(APP_STORAGE_URI, ''); + } else { + try { + await NativeFile.copyFile( + cover.split(/[?#]/, 1)[0], + coversDirPath + '/' + novel.id, + ); + cover = cover.replace(APP_STORAGE_URI, ''); + } catch { + cover = null; + } + } + } + const preparedNovel = { + ...novel, + chapters: backedUpChapters, + cover, + }; + return validateBackupNovel(preparedNovel); + } catch { + failedNovelCount++; + return null; + } + }), + ); + preparedNovels.push( + ...prepared.filter((novel): novel is BackupNovel => novel !== null), + ); + } + + if (preparedNovels.length > 0) { + const batchName = `batch-${String( + Math.floor(start / BACKUP_NOVEL_BATCH_SIZE) + 1, + ).padStart(6, '0')}.json`; + const batchPath = novelDirPath + '/' + batchName; + try { + await NativeFile.writeFile( + batchPath, + JSON.stringify(encodeNovelBatch(preparedNovels)), + ); + backedUpNovelCount += preparedNovels.length; + } catch { + failedNovelCount += preparedNovels.length; + try { + await NativeFile.unlink(batchPath); + } catch { + // Best effort cleanup prevents a failed write from restoring partial data. + } + } + } + } + manifest.novelCount = backedUpNovelCount; + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + JSON.stringify(manifest), + ); + + // categories + try { + const categories = await getCategoriesFromDb(); + const novelCategories = await getAllNovelCategories(); + const novelIdsByCategory = new Map(); + for (const novelCategory of novelCategories) { + const novelIds = novelIdsByCategory.get(novelCategory.categoryId); + if (novelIds) { + novelIds.push(novelCategory.novelId); + } else { + novelIdsByCategory.set(novelCategory.categoryId, [ + novelCategory.novelId, + ]); + } + } + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.CATEGORY, + JSON.stringify( + categories.map(category => ({ + ...category, + novelIds: novelIdsByCategory.get(category.id) ?? [], + })), + ), + ); + } catch { + failedSectionCount++; + } + } + + // settings + if (options.settings) { + try { + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.SETTING, + JSON.stringify(backupMMKVData()), + ); + } catch { + failedSectionCount++; + } + } + + // installed plugin registry + if (options.plugins) { + try { + await NativeFile.writeFile( + cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA, + MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', + ); + } catch { + failedSectionCount++; + } + } + + return { + failedNovelCount, + failedSectionCount, + }; +}; diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index d00e13041c..ae3c475d54 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -1,31 +1,16 @@ import { DriveFile } from '@api/drive/types'; import { sleep } from '@utils/sleep'; -import { exists } from '@api/drive'; import { getString } from '@i18n/translations'; -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; -import { download, updateMetadata, uploadMedia } from '@api/drive/request'; +import { updateMetadata, uploadMedia } from '@api/drive/request'; import { ZipBackupName } from '../types'; import type { DriveBackupData, TaskProgressUpdater, } from '@services/backgroundTasks/contracts'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions } from '../options'; const uploadBackupSection = async ( @@ -77,7 +62,7 @@ export const createDriveBackup = async ( progressText: getString('backupScreen.uploadingSelectedFiles'), })); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await uploadBackupSection( section.storagePath, section.archiveName, @@ -94,91 +79,3 @@ export const createDriveBackup = async ( completionText, })); }; - -export const driveRestore = async ( - backupFolder: DriveFile, - setMeta: TaskProgressUpdater, -) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - const zipDataFile = await exists(ZipBackupName.DATA, false, backupFolder.id); - if (!zipDataFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - - await clearBackupCache(); - await download(zipDataFile, CACHE_DIR_PATH); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFile = await exists( - ZipBackupName.DOWNLOAD, - false, - backupFolder.id, - ); - if (!legacyFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download(legacyFile, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - const file = await exists(section.archiveName, false, backupFolder.id); - if (!file) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - await download( - file, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); -}; diff --git a/src/services/backup/fileSections.ts b/src/services/backup/fileSections.ts index ece55687ad..dad9a6e2df 100644 --- a/src/services/backup/fileSections.ts +++ b/src/services/backup/fileSections.ts @@ -1,9 +1,6 @@ import { NOVEL_STORAGE, PLUGIN_STORAGE } from '@utils/Storages'; import type { BackupOptions } from './options'; import { ZipBackupName } from './types'; -import NativeFile from '@modules/native-file'; -import type { RestoredNovelMapping } from '@database/types'; - export type BackupFileSection = { archiveName: ZipBackupName; storagePath: string; @@ -11,6 +8,7 @@ export type BackupFileSection = { export const getSelectedBackupFileSections = ( options: BackupOptions, + formatVersion: 2 | 3 = 3, ): BackupFileSection[] => { const sections: BackupFileSection[] = []; @@ -20,7 +18,7 @@ export const getSelectedBackupFileSections = ( storagePath: PLUGIN_STORAGE, }); } - if (options.downloadedFiles) { + if (formatVersion === 2 && options.downloadedFiles) { sections.push({ archiveName: ZipBackupName.NOVEL_FILES, storagePath: NOVEL_STORAGE, @@ -29,78 +27,3 @@ export const getSelectedBackupFileSections = ( return sections; }; - -export const getNovelFilesRestorePath = (cacheDirPath: string) => - `${cacheDirPath}/RestoredNovelFiles`; - -export const getLegacyFilesRestorePath = (cacheDirPath: string) => - `${cacheDirPath}/RestoredLegacyFiles`; - -const moveDirectoryContents = async (source: string, destination: string) => { - if (!(await NativeFile.exists(source))) { - return; - } - await NativeFile.mkdir(destination); - for (const item of await NativeFile.readDir(source)) { - const destinationPath = `${destination}/${item.name}`; - if (item.isDirectory) { - await moveDirectoryContents(item.path, destinationPath); - } else { - await NativeFile.moveFile(item.path, destinationPath); - } - } -}; - -export const restoreNovelFiles = async ( - stagingPath: string, - novelMappings: RestoredNovelMapping[], -) => { - for (const mapping of novelMappings) { - const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; - if (!(await NativeFile.exists(sourceNovelPath))) { - continue; - } - - const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; - await NativeFile.mkdir(destinationNovelPath); - const chapterIds = new Map( - mapping.chapters.map(chapter => [ - String(chapter.backupChapterId), - chapter.restoredChapterId, - ]), - ); - - for (const item of await NativeFile.readDir(sourceNovelPath)) { - if (!item.isDirectory) { - await NativeFile.moveFile( - item.path, - `${destinationNovelPath}/${item.name}`, - ); - continue; - } - - const restoredChapterId = chapterIds.get(item.name); - if (restoredChapterId !== undefined) { - await moveDirectoryContents( - item.path, - `${destinationNovelPath}/${restoredChapterId}`, - ); - } - } - } - - if (await NativeFile.exists(stagingPath)) { - await NativeFile.unlink(stagingPath); - } -}; - -export const restoreLegacyFiles = async ( - stagingPath: string, - novelMappings: RestoredNovelMapping[], -) => { - await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); - await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings); - if (await NativeFile.exists(stagingPath)) { - await NativeFile.unlink(stagingPath); - } -}; diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index 85e7822c32..2ede9d2776 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -1,27 +1,14 @@ -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; import NativeZipArchive from '@modules/native-zip-archive'; -import { ZipBackupName } from '../types'; +import { BackupEntryName } from '../types'; import NativeFile from '@modules/native-file'; import { getString } from '@i18n/translations'; import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; import { sleep } from '@utils/sleep'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; export const createBackup = async ( @@ -40,7 +27,7 @@ export const createBackup = async ( progressText: getString('backupScreen.preparingData'), })); - const backupResult = await prepareBackupData(CACHE_DIR_PATH, options); + const backupResult = await prepareBackupData(CACHE_DIR_PATH, options, 3); setMeta?.(meta => ({ ...meta, @@ -65,7 +52,15 @@ export const createBackup = async ( await sleep(200); - await NativeZipArchive.zip(CACHE_DIR_PATH, CACHE_DIR_PATH + '.zip'); + await NativeZipArchive.zipDirectories( + [ + { path: CACHE_DIR_PATH, prefix: '' }, + ...(options.downloadedFiles + ? [{ path: NOVEL_STORAGE, prefix: BackupEntryName.NOVEL_FILES }] + : []), + ], + CACHE_DIR_PATH + '.zip', + ); setMeta?.(meta => ({ ...meta, @@ -83,107 +78,7 @@ export const createBackup = async ( progressText: completionText, completionText, })); - } catch (error: any) { - setMeta?.(meta => ({ - ...meta, - isRunning: false, - })); - throw error; - } -}; - -export const restoreBackup = async ( - { sourceUri }: { sourceUri: string }, - setMeta?: TaskProgressUpdater, -) => { - try { - setMeta?.(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 4, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - const localPath = CACHE_DIR_PATH + '-source.zip'; - await NativeFile.copyFile(sourceUri, localPath); - - setMeta?.(meta => ({ - ...meta, - progress: 1 / 4, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - await NativeZipArchive.unzip(localPath, CACHE_DIR_PATH); - - setMeta?.(meta => ({ - ...meta, - progress: 2 / 4, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta?.(meta => ({ - ...meta, - progress: 3 / 4, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyArchive = CACHE_DIR_PATH + '/' + ZipBackupName.DOWNLOAD; - if (!(await NativeFile.exists(legacyArchive))) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await NativeZipArchive.unzip(legacyArchive, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - const archivePath = `${CACHE_DIR_PATH}/${section.archiveName}`; - if (!(await NativeFile.exists(archivePath))) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - await NativeZipArchive.unzip( - archivePath, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta?.(meta => ({ - ...meta, - progress: 4 / 4, - isRunning: false, - progressText: completionText, - completionText, - })); - } catch (error: any) { + } catch (error) { setMeta?.(meta => ({ ...meta, isRunning: false, diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts new file mode 100644 index 0000000000..b8c00c2102 --- /dev/null +++ b/src/services/backup/novelPayload.ts @@ -0,0 +1,444 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; + +type NullableString = string | null; +type NullableNumber = number | null; +type NullableBoolean = boolean | null; + +type CompactChapter = [ + number, + string, + string, + NullableString, + NullableBoolean, + NullableBoolean, + NullableString, + NullableBoolean, + NullableString, + NullableNumber, + NullableString, + NullableNumber, + NullableNumber, + NullableString, + NullableNumber, +]; + +type CompactNovel = { + c: CompactChapter[]; + id: number; + p: string; + pi: string; + n: string; + co: NullableString; + s: NullableString; + a: NullableString; + ar: NullableString; + st: NullableString; + g: NullableString; + l: NullableBoolean; + lo: NullableBoolean; + t: NullableNumber; + d: NullableNumber; + u: NullableNumber; + tc: NullableNumber; + lr: NullableString; + lu: NullableString; +}; + +type BackupNovelWithAggregates = BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; +}; + +const NOVEL_KEYS = [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', +] as const; +const EXPECTED_NOVEL_KEYS = [...NOVEL_KEYS].sort(); + +const isNullable = ( + value: unknown, + predicate: (value: unknown) => value is T, +): value is T | null => value === null || predicate(value); + +const isString = (value: unknown): value is string => typeof value === 'string'; +const isNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); +const isId = (value: unknown): value is number => + isNumber(value) && Number.isInteger(value) && value > 0; +const isBoolean = (value: unknown): value is boolean => + typeof value === 'boolean'; + +const nullableString = (value: unknown): value is NullableString => + isNullable(value, isString); +const nullableNumber = (value: unknown): value is NullableNumber => + isNullable(value, isNumber); +const nullableBoolean = (value: unknown): value is NullableBoolean => + isNullable(value, isBoolean); +const nonEmptyString = (value: unknown): value is string => + isString(value) && value.trim().length > 0; + +const nullableLegacyString = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isString(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyNumber = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isNumber(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyBoolean = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isBoolean(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const positiveId = (value: unknown): value is number => + isId(value) && value > 0; + +const normalizeLegacyChapter = ( + value: unknown, + containingNovelId: number, +): ChapterInfo => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy chapter'); + } + const chapter = value as Record; + if ( + !positiveId(chapter.id) || + !nonEmptyString(chapter.path) || + !isString(chapter.name) || + (chapter.novelId !== undefined && + chapter.novelId !== null && + !positiveId(chapter.novelId)) + ) { + throw new Error('Invalid legacy chapter fields'); + } + return { + id: chapter.id, + novelId: containingNovelId, + path: chapter.path, + name: chapter.name, + releaseTime: nullableLegacyString(chapter, 'releaseTime'), + readTime: nullableLegacyString(chapter, 'readTime'), + bookmark: nullableLegacyBoolean(chapter, 'bookmark'), + unread: nullableLegacyBoolean(chapter, 'unread'), + isDownloaded: nullableLegacyBoolean(chapter, 'isDownloaded'), + updatedTime: nullableLegacyString(chapter, 'updatedTime'), + chapterNumber: nullableLegacyNumber(chapter, 'chapterNumber'), + page: nullableLegacyString(chapter, 'page'), + progress: nullableLegacyNumber(chapter, 'progress'), + position: nullableLegacyNumber(chapter, 'position'), + scanlator: nullableLegacyString(chapter, 'scanlator'), + timeSpent: nullableLegacyNumber(chapter, 'timeSpent'), + }; +}; + +export const validateBackupNovel = (novel: BackupNovel): BackupNovel => { + if ( + !positiveId(novel.id) || + !nonEmptyString(novel.path) || + !nonEmptyString(novel.pluginId) || + !isString(novel.name) || + !Array.isArray(novel.chapters) + ) { + throw new Error('Invalid novel fields'); + } + const stringFields = [ + novel.cover, + novel.summary, + novel.author, + novel.artist, + novel.status, + novel.genres, + ]; + if (stringFields.some(value => value !== null && !isString(value))) { + throw new Error('Invalid novel string field'); + } + const booleanFields = [novel.inLibrary, novel.isLocal]; + if (booleanFields.some(value => value !== null && !isBoolean(value))) { + throw new Error('Invalid novel boolean field'); + } + if (novel.totalPages !== null && !isNumber(novel.totalPages)) { + throw new Error('Invalid novel number field'); + } + for (const chapter of novel.chapters) { + if ( + !positiveId(chapter.id) || + !positiveId(chapter.novelId) || + !nonEmptyString(chapter.path) || + !isStringOrNull(chapter.releaseTime) || + !isStringOrNull(chapter.readTime) || + !isBooleanOrNull(chapter.bookmark) || + !isBooleanOrNull(chapter.unread) || + !isBooleanOrNull(chapter.isDownloaded) || + !isStringOrNull(chapter.updatedTime) || + !isNumberOrNull(chapter.chapterNumber) || + !isStringOrNull(chapter.page) || + !isNumberOrNull(chapter.progress) || + !isNumberOrNull(chapter.position) || + !isStringOrNull(chapter.scanlator) || + !isNumberOrNull(chapter.timeSpent) + ) { + throw new Error('Invalid chapter fields'); + } + } + return novel; +}; + +const isStringOrNull = (value: unknown): value is string | null => + value === null || isString(value); +const isNumberOrNull = (value: unknown): value is number | null => + value === null || isNumber(value); +const isBooleanOrNull = (value: unknown): value is boolean | null => + value === null || isBoolean(value); + +export const normalizeLegacyNovel = (value: unknown): BackupNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy novel'); + } + const source = value as Record; + if ( + !positiveId(source.id) || + !nonEmptyString(source.path) || + !nonEmptyString(source.pluginId) || + !isString(source.name) || + !Array.isArray(source.chapters) + ) { + throw new Error('Invalid legacy novel fields'); + } + const novelId = source.id; + const novel: BackupNovel = { + id: novelId, + path: source.path, + pluginId: source.pluginId, + name: source.name, + cover: nullableLegacyString(source, 'cover'), + summary: nullableLegacyString(source, 'summary'), + author: nullableLegacyString(source, 'author'), + artist: nullableLegacyString(source, 'artist'), + status: nullableLegacyString(source, 'status'), + genres: nullableLegacyString(source, 'genres'), + inLibrary: nullableLegacyBoolean(source, 'inLibrary'), + isLocal: nullableLegacyBoolean(source, 'isLocal'), + totalPages: nullableLegacyNumber(source, 'totalPages'), + chapters: source.chapters.map(chapter => + normalizeLegacyChapter(chapter, novelId), + ), + }; + return novel; +}; + +const assertNovelKeys = (novel: Record) => { + const keys = Object.keys(novel).sort(); + const expected = EXPECTED_NOVEL_KEYS; + if ( + keys.length !== expected.length || + keys.some((key, index) => key !== expected[index]) + ) { + throw new Error('Invalid compact novel keys'); + } +}; + +const isCompactChapter = (value: unknown): value is CompactChapter => { + if (!Array.isArray(value) || value.length !== 15) { + return false; + } + + return ( + isId(value[0]) && + nonEmptyString(value[1]) && + isString(value[2]) && + nullableString(value[3]) && + nullableBoolean(value[4]) && + nullableBoolean(value[5]) && + nullableString(value[6]) && + nullableBoolean(value[7]) && + nullableString(value[8]) && + nullableNumber(value[9]) && + nullableString(value[10]) && + nullableNumber(value[11]) && + nullableNumber(value[12]) && + nullableString(value[13]) && + nullableNumber(value[14]) + ); +}; + +type CompactNovelHeader = Omit & { c: unknown[] }; + +const isCompactNovelHeader = (value: unknown): value is CompactNovelHeader => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const novel = value as Record; + assertNovelKeys(novel); + return ( + Array.isArray(novel.c) && + isId(novel.id) && + nonEmptyString(novel.p) && + nonEmptyString(novel.pi) && + isString(novel.n) && + nullableString(novel.co) && + nullableString(novel.s) && + nullableString(novel.a) && + nullableString(novel.ar) && + nullableString(novel.st) && + nullableString(novel.g) && + nullableBoolean(novel.l) && + nullableBoolean(novel.lo) && + nullableNumber(novel.t) && + nullableNumber(novel.d) && + nullableNumber(novel.u) && + nullableNumber(novel.tc) && + nullableString(novel.lr) && + nullableString(novel.lu) + ); +}; + +const nullable = (value: T | undefined | null): T | null => value ?? null; + +const encodeChapter = (chapter: ChapterInfo): CompactChapter => [ + chapter.id, + chapter.path, + chapter.name, + nullable(chapter.releaseTime), + nullable(chapter.bookmark), + nullable(chapter.unread), + nullable(chapter.readTime), + nullable(chapter.isDownloaded), + nullable(chapter.updatedTime), + nullable(chapter.chapterNumber), + nullable(chapter.page), + nullable(chapter.position), + nullable(chapter.progress), + nullable(chapter.scanlator), + nullable(chapter.timeSpent), +]; + +const encodeNovel = (novel: BackupNovel): CompactNovel => { + const novelWithAggregates = novel as BackupNovelWithAggregates; + return { + c: novel.chapters.map(encodeChapter), + id: novel.id, + p: novel.path, + pi: novel.pluginId, + n: novel.name, + co: nullable(novel.cover), + s: nullable(novel.summary), + a: nullable(novel.author), + ar: nullable(novel.artist), + st: nullable(novel.status), + g: nullable(novel.genres), + l: nullable(novel.inLibrary), + lo: nullable(novel.isLocal), + t: nullable(novel.totalPages), + d: nullable(novelWithAggregates.chaptersDownloaded), + u: nullable(novelWithAggregates.chaptersUnread), + tc: nullable(novelWithAggregates.totalChapters), + lr: nullable(novelWithAggregates.lastReadAt), + lu: nullable(novelWithAggregates.lastUpdatedAt), + }; +}; + +const decodeNovel = (novel: CompactNovelHeader): BackupNovel => { + const decoded: BackupNovelWithAggregates = { + id: novel.id, + path: novel.p, + pluginId: novel.pi, + name: novel.n, + cover: novel.co, + summary: novel.s, + author: novel.a, + artist: novel.ar, + status: novel.st, + genres: novel.g, + inLibrary: novel.l, + isLocal: novel.lo, + totalPages: novel.t, + chaptersDownloaded: novel.d, + chaptersUnread: novel.u, + totalChapters: novel.tc, + lastReadAt: novel.lr, + lastUpdatedAt: novel.lu, + chapters: novel.c.map(chapter => { + if (!isCompactChapter(chapter)) { + throw new Error('Invalid compact novel record'); + } + return { + id: chapter[0], + novelId: novel.id, + path: chapter[1], + name: chapter[2], + releaseTime: chapter[3], + bookmark: chapter[4], + unread: chapter[5], + isDownloaded: chapter[7], + updatedTime: chapter[8], + chapterNumber: chapter[9], + page: chapter[10], + position: chapter[11], + progress: chapter[12], + scanlator: chapter[13], + timeSpent: chapter[14], + readTime: chapter[6], + }; + }), + }; + return decoded; +}; + +export const encodeNovelBatch = (novels: BackupNovel[]): CompactNovel[] => { + if (novels.length > 100) { + throw new Error('Compact novel batch exceeds 100 novels'); + } + return novels.map(encodeNovel); +}; + +export const decodeNovelBatch = (payload: unknown): BackupNovel[] => { + if (!Array.isArray(payload) || payload.length > 100) { + throw new Error('Invalid compact novel batch'); + } + return payload.map(value => { + if (!isCompactNovelHeader(value)) { + throw new Error('Invalid compact novel record'); + } + return decodeNovel(value); + }); +}; diff --git a/src/services/backup/restore/__tests__/drive.test.ts b/src/services/backup/restore/__tests__/drive.test.ts new file mode 100644 index 0000000000..fb115c0222 --- /dev/null +++ b/src/services/backup/restore/__tests__/drive.test.ts @@ -0,0 +1,240 @@ +import { exists } from '@api/drive'; +import type { DriveFile } from '@api/drive/types'; +import { download } from '@api/drive/request'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { CACHE_DIR_PATH, clearBackupCache } from '../../cache'; +import { ZipBackupName } from '../../types'; +import { restoreNovelFiles } from '../files'; + +import { driveRestore } from '../drive'; +import { restoreData } from '../index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from '../result'; +import type { RestoreProgressReporter } from '../progress'; + +jest.mock('@api/drive', () => ({ exists: jest.fn() })); +jest.mock('@api/drive/request', () => ({ download: jest.fn() })); +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); +jest.mock('../files', () => ({ + getLegacyFilesRestorePath: jest.fn(() => '/cache/legacy-files'), + getNovelFilesRestorePath: jest.fn(() => '/cache/novel-files'), + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), +})); +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(), + restoreData: jest.fn(), +})); +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(() => 'completed'), +})); +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +const dataFile = { id: 'data-file', name: 'data.zip' } as DriveFile; +const backupFolder = { id: 'backup-folder', name: 'backup' } as DriveFile; + +const createRestoreResult = ( + plugins: boolean, + downloadedFiles = false, + formatVersion: 1 | 2 | 3 = 3, + novelMappings: RestoreResult['novelMappings'] = [], +): RestoreResult => ({ + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: plugins ? ['restored-plugin'] : [], + novelMappings, + restoreRunId: 'drive-restore-run', + manifest: { + appVersion: '2.1.3', + formatVersion, + novelDataFormat: 2, + sections: { + library: downloadedFiles, + settings: false, + plugins, + downloadedFiles, + }, + }, +}); + +const reportRestoreStages = (reporter: RestoreProgressReporter | undefined) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 1, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'DRIVE_RESTORE', + isRunning: false, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta, getMetadata: () => metadata }; +}; + +describe('Drive restore progress', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(exists).mockResolvedValue(dataFile); + jest.mocked(download).mockResolvedValue(''); + jest.mocked(clearBackupCache).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValue([]); + }); + + it('keeps remote retrieval and later phases monotonic through completion', async () => { + const result = createRestoreResult(false); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await driveRestore(backupFolder, capture.setMeta); + + expect(capture.progressValues[0]).toBe(0); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + expect( + capture.progressValues.every( + (value, index) => + index === 0 || value >= capture.progressValues[index - 1], + ), + ).toBe(true); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + expect(download).toHaveBeenCalledWith(dataFile, CACHE_DIR_PATH); + }); + + it('does not report completion when a selected archive is missing', async () => { + const result = createRestoreResult(true); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + jest + .mocked(exists) + .mockImplementation(async name => + name === ZipBackupName.DATA ? dataFile : undefined, + ); + const capture = createProgressCapture(); + + await expect(driveRestore(backupFolder, capture.setMeta)).rejects.toThrow( + 'backupScreen.invalidBackupFolder', + ); + + expect(capture.progressValues.length).toBeGreaterThan(0); + expect(Math.max(...capture.progressValues)).toBeLessThan(1); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + it('restores selected format-2 archives before finalizing plugins', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = createRestoreResult(true, true, 2, mappings); + const pluginFile = { + id: 'plugins-file', + name: ZipBackupName.PLUGINS, + } as DriveFile; + const novelFilesFile = { + id: 'novel-files-file', + name: ZipBackupName.NOVEL_FILES, + } as DriveFile; + const files: Record = { + [ZipBackupName.DATA]: dataFile, + [ZipBackupName.PLUGINS]: pluginFile, + [ZipBackupName.NOVEL_FILES]: novelFilesFile, + }; + jest.mocked(exists).mockImplementation(async name => files[name]); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await driveRestore(backupFolder, capture.setMeta); + + expect(download).toHaveBeenCalledWith(dataFile, CACHE_DIR_PATH); + expect(download).toHaveBeenCalledWith(pluginFile, '/storage/Plugins'); + expect(download).toHaveBeenCalledWith(novelFilesFile, '/cache/novel-files'); + expect(restoreNovelFiles).toHaveBeenCalledWith( + '/cache/novel-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(result); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ); + expect( + jest.mocked(getRestoreCompletionText).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + }); + + it('rejects a missing data archive before clearing the cache or downloading', async () => { + jest.mocked(exists).mockResolvedValue(undefined); + const capture = createProgressCapture(); + + await expect(driveRestore(backupFolder, capture.setMeta)).rejects.toThrow( + 'backupScreen.invalidBackupFolder', + ); + + expect(exists).toHaveBeenCalledWith( + ZipBackupName.DATA, + false, + backupFolder.id, + ); + expect(clearBackupCache).not.toHaveBeenCalled(); + expect(download).not.toHaveBeenCalled(); + expect(restoreData).not.toHaveBeenCalled(); + }); +}); diff --git a/src/services/backup/restore/__tests__/files.test.ts b/src/services/backup/restore/__tests__/files.test.ts new file mode 100644 index 0000000000..246602eef5 --- /dev/null +++ b/src/services/backup/restore/__tests__/files.test.ts @@ -0,0 +1,220 @@ +import NativeFile from '@modules/native-file'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { restoreLegacyFiles, restoreNovelFiles } from '../files'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +describe('restore file sections', () => { + beforeEach(() => { + jest.mocked(getRestoreChapterMappings).mockReset(); + }); + + it('moves downloaded files with bounded staged mapping lookups', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + const chapterItems = Array.from({ length: 101 }, (_, index) => ({ + name: String(index + 1), + path: `/staging/source/1/${index + 1}`, + isDirectory: true, + })); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/staging/source/1') { + return [ + { + name: 'cover.png', + path: '/staging/source/1/cover.png', + isDirectory: false, + }, + ...chapterItems, + ]; + } + if (path === '/staging/source/1/10') { + return [ + { + name: 'index.html', + path: '/staging/source/1/10/index.html', + isDirectory: false, + }, + ]; + } + if (path === '/staging/source/1/101') { + return [ + { + name: 'index.html', + path: '/staging/source/1/101/index.html', + isDirectory: false, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockImplementation( + async (_restoreRunId, _backupNovelId, backupChapterIds) => + backupChapterIds + .filter(id => id === 10 || id === 101) + .map(backupChapterId => ({ + backupChapterId, + restoredChapterId: backupChapterId === 10 ? 99 : 199, + })), + ); + + const progress: { completed: number; total: number }[] = []; + await restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run-1', + (completed, total) => progress.push({ completed, total }), + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledTimes(2); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 1, + 'restore-run-1', + 1, + Array.from({ length: 100 }, (_, index) => index + 1), + ); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 2, + 'restore-run-1', + 1, + [101], + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/cover.png', + '/storage/Novels/source/7/cover.png', + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/10/index.html', + '/storage/Novels/source/7/99/index.html', + ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/101/index.html', + '/storage/Novels/source/7/199/index.html', + ); + expect(NativeFile.moveFile).not.toHaveBeenCalledWith( + expect.stringContaining('/20/'), + expect.any(String), + ); + expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); + expect(progress).toEqual([ + { completed: 1, total: 2 }, + { completed: 2, total: 2 }, + ]); + }); + + it('counts missing novel directories and only completes after staging cleanup', async () => { + jest.mocked(NativeFile.exists).mockImplementation(async path => { + if (path === '/staging') { + return true; + } + return false; + }); + jest + .mocked(NativeFile.unlink) + .mockRejectedValue(new Error('staging cleanup failed')); + const progress: { completed: number; total: number }[] = []; + + await expect( + restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run', + (completed, total) => progress.push({ completed, total }), + ), + ).rejects.toThrow('staging cleanup failed'); + + expect(progress).toEqual([{ completed: 1, total: 2 }]); + }); + + it('reports the staging cleanup for an empty mapping set', async () => { + const events: string[] = []; + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.unlink).mockImplementation(async () => { + events.push('cleanup'); + }); + const progress: { completed: number; total: number }[] = []; + + await restoreNovelFiles( + '/staging', + [], + 'restore-run', + (completed, total) => { + progress.push({ completed, total }); + events.push(`progress:${completed}/${total}`); + }, + ); + + expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); + expect(progress).toEqual([{ completed: 1, total: 1 }]); + expect(events).toEqual(['cleanup', 'progress:1/1']); + }); + + it('passes the restore run ID through legacy file restoration', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/legacy/Novels/source/1') { + return [ + { + name: '10', + path: '/legacy/Novels/source/1/10', + isDirectory: true, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockResolvedValue([{ backupChapterId: 10, restoredChapterId: 99 }]); + + const progress: { completed: number; total: number }[] = []; + await restoreLegacyFiles( + '/legacy', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'legacy-run', + (completed, total) => progress.push({ completed, total }), + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledWith('legacy-run', 1, [ + 10, + ]); + expect(NativeFile.unlink).toHaveBeenCalledWith('/legacy'); + expect(progress).toEqual([ + { completed: 1, total: 3 }, + { completed: 2, total: 3 }, + { completed: 3, total: 3 }, + ]); + }); +}); diff --git a/src/services/backup/restore/__tests__/index.test.ts b/src/services/backup/restore/__tests__/index.test.ts new file mode 100644 index 0000000000..e2bb4ea97d --- /dev/null +++ b/src/services/backup/restore/__tests__/index.test.ts @@ -0,0 +1,1282 @@ +import { + clearRestoreChapterMappings, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import NativeFile from '@modules/native-file'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { clearRestoreChapterMappingsSafely, restoreData } from '../index'; +import { encodeNovelBatch } from '../../novelPayload'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; +import type { BackupOptions } from '../../options'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(async () => undefined), + _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), +})); + +jest.mock('@database/queries/CategoryQueries', () => ({ + _restoreCategory: jest.fn(), +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const pluginOnlyOptions: BackupOptions = { + library: false, + settings: false, + plugins: true, + downloadedFiles: false, +}; +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +describe('selective backup data', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(NativeFile.exists).mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + const restoreNovel = async ( + novel: BackupNovel, + _options?: { includeChapterMappings?: boolean; restoreRunId?: string }, + ): Promise => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id, + }); + jest.mocked(_restoreNovelAndChapters).mockImplementation(restoreNovel); + jest.mocked(_restoreNovelsAndChapters).mockImplementation( + async ( + novels: BackupNovel[], + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => Promise.all(novels.map(novel => restoreNovel(novel, _options))), + ); + }); + + it('accepts v3 section manifests', async () => { + const options: BackupOptions = { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }), + ); + + const result = await restoreData('/cache'); + + expect(result.manifest).toEqual({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + + it('does not warn about sections intentionally omitted by the manifest', async () => { + jest + .mocked(NativeFile.readFile) + .mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 2, + sections: pluginOnlyOptions, + }), + ) + .mockResolvedValueOnce('[]'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path.endsWith('/Plugins.json')); + + const result = await restoreData('/cache'); + + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + failedNovelCount: 0, + failedCategoryCount: 0, + failedSectionCount: 0, + settingsRestored: true, + manifest: { + formatVersion: 2, + sections: pluginOnlyOptions, + }, + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(_restoreCategory).not.toHaveBeenCalled(); + expect(MMKVStorage.set).toHaveBeenCalledWith('INSTALL_PLUGINS', '[]'); + }); + + it('merges restored plugins with the existing registry', async () => { + const options: BackupOptions = { + library: false, + settings: true, + plugins: true, + downloadedFiles: false, + }; + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => + path.endsWith('/Version.json') || + path.endsWith('/Setting.json') || + path.endsWith('/Plugins.json'), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ + INSTALL_PLUGINS: JSON.stringify([ + { id: 'restored', name: 'Restored' }, + ]), + THEME: 'dark', + }); + } + return JSON.stringify([{ id: 'restored', name: 'Restored' }]); + }); + jest + .mocked(MMKVStorage.getString) + .mockReturnValueOnce( + JSON.stringify([{ id: 'existing', name: 'Existing' }]), + ); + + await restoreData('/cache'); + + expect(MMKVStorage.set).toHaveBeenCalledWith('THEME', 'dark'); + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([ + { id: 'existing', name: 'Existing' }, + { id: 'restored', name: 'Restored' }, + ]), + ); + }); + it.each(['{}', 'null'])( + 'isolates malformed category payload %s from other restore sections', + async categoryPayload => { + const options: BackupOptions = { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/Version.json', + '/cache/NovelAndChapters', + '/cache/Category.json', + '/cache/Setting.json', + '/cache/Plugins.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 3, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return categoryPayload; + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ reviewSetting: true }); + } + return JSON.stringify([{ id: 'review-plugin', name: 'Review plugin' }]); + }); + + const result = await restoreData('/cache'); + + expect(result).toMatchObject({ + failedSectionCount: 1, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + }); + expect(_restoreCategory).not.toHaveBeenCalled(); + expect(MMKVStorage.set).toHaveBeenCalledWith('reviewSetting', true); + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([{ id: 'review-plugin', name: 'Review plugin' }]), + ); + }, + ); + + it('propagates category progress interruptions before restoring settings', async () => { + const options: BackupOptions = { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }; + const interruption = new Error('category progress interrupted'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/Version.json', + '/cache/NovelAndChapters', + '/cache/Category.json', + '/cache/Setting.json', + '/cache/Plugins.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.0', + formatVersion: 3, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return '[]'; + } + if (path.endsWith('/Setting.json')) { + return JSON.stringify({ reviewSetting: true }); + } + return '[]'; + }); + const progressReporter = jest.fn((phase: string) => { + if (phase === 'categories') { + throw interruption; + } + }); + + await expect(restoreData('/cache', progressReporter)).rejects.toBe( + interruption, + ); + expect(MMKVStorage.set).not.toHaveBeenCalledWith('reviewSetting', true); + }); + + it('merges the plugin registry from legacy settings', async () => { + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path.endsWith('/Setting.json')); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ version: '2.0.0' }); + } + return JSON.stringify({ + INSTALL_PLUGINS: JSON.stringify([{ id: 'legacy', name: 'Legacy' }]), + }); + }); + jest + .mocked(MMKVStorage.getString) + .mockReturnValueOnce( + JSON.stringify([{ id: 'existing', name: 'Existing' }]), + ); + + await restoreData('/cache'); + + expect(MMKVStorage.set).toHaveBeenCalledWith( + 'INSTALL_PLUGINS', + JSON.stringify([ + { id: 'existing', name: 'Existing' }, + { id: 'legacy', name: 'Legacy' }, + ]), + ); + }); + + it('restores stored covers from library data and preserves missing covers', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.2', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/1.json')) { + return JSON.stringify({ + id: 1, + name: 'Stored cover', + path: '/stored-cover', + pluginId: 'source', + cover: '/Novels/source/1/cover.png?123', + chapters: [], + }); + } + if (path.endsWith('/2.json')) { + return JSON.stringify({ + id: 2, + name: 'Missing cover', + path: '/missing-cover', + pluginId: 'source', + cover: null, + chapters: [], + }); + } + return '[]'; + }); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: '1.json', + path: '/cache/NovelAndChapters/1.json', + isDirectory: false, + }, + { + name: '2.json', + path: '/cache/NovelAndChapters/2.json', + isDirectory: false, + }, + ]); + + await restoreData('/cache'); + + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/1', + '/storage/Novels/source/1/cover.png', + ); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 1, + cover: 'file:///storage/Novels/source/1/cover.png?123', + }), + expect.objectContaining({ id: 2, cover: null }), + ], + { includeChapterMappings: false }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + }); + + it('keeps restored mappings when a stored cover copy fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novels = [1, 2, 3].map(id => ({ + ...makeTestNovel(id), + cover: `/Novels/source/${id}/cover.png`, + })); + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + const category = { id: 1, name: 'Category', novelIds: [1, 2, 3] }; + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Covers/2', + '/cache/Covers/3', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return JSON.stringify([category]); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + if (!novel) { + throw new Error(`Unexpected read: ${path}`); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + jest.mocked(NativeFile.copyFile).mockImplementation(async source => { + if (source === '/cache/Covers/1') { + throw new Error('Cover copy failed'); + } + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + novels.map(novel => ({ + ...novel, + cover: `file:///storage${novel.cover}`, + })), + { includeChapterMappings: false }, + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/2', + '/storage/Novels/source/102/cover.png', + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/3', + '/storage/Novels/source/103/cover.png', + ); + expect(_restoreCategory).toHaveBeenCalledWith( + expect.objectContaining({ novelIds: [1, 2, 3] }), + new Map([ + [1, 101], + [2, 102], + [3, 103], + ]), + ); + expect(result).toMatchObject({ + novelCount: 3, + failedNovelCount: 1, + failedSectionCount: 0, + novelMappings: mappings, + }); + }); + + it('restores downloaded-file novels in one mapping-aware batch', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 11, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 101, + novelId: 11, + name: 'First chapter', + path: '/first/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 22, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 202, + novelId: 22, + name: 'Second chapter', + path: '/second/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ]; + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'source', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'source', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result.restoreRunId).toBe( + jest.mocked(_restoreNovelsAndChapters).mock.calls[0][1]?.restoreRunId, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it('restores scrambled compact batches in order with downloaded-file mappings', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const firstNovel = makeTestNovel(11, 'plugin-a', [makeTestChapter(11, 1)]); + const secondNovel = makeTestNovel(22, 'plugin-b', [makeTestChapter(22, 1)]); + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'plugin-a', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'plugin-b', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'batch-000002.json', + path: '/cache/NovelAndChapters/batch-000002.json', + isDirectory: false, + }, + { + name: 'ignored-directory', + path: '/cache/NovelAndChapters/ignored-directory', + isDirectory: true, + }, + { + name: 'batch-000001.json', + path: '/cache/NovelAndChapters/batch-000001.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + if (path.endsWith('/batch-000001.json')) { + return JSON.stringify(encodeNovelBatch([firstNovel])); + } + if (path.endsWith('/batch-000002.json')) { + return JSON.stringify(encodeNovelBatch([secondNovel])); + } + throw new Error(`Unexpected read: ${path}`); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + const restoredNovels = jest.mocked(_restoreNovelsAndChapters).mock + .calls[0][0] as BackupNovel[]; + expect(restoredNovels.map(novel => novel.id)).toEqual([11, 22]); + expect( + restoredNovels.map(novel => + novel.chapters.map(chapter => ({ + id: chapter.id, + novelId: chapter.novelId, + isDownloaded: chapter.isDownloaded, + })), + ), + ).toEqual([ + [{ id: 1101, novelId: 11, isDownloaded: true }], + [{ id: 2201, novelId: 22, isDownloaded: true }], + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(expect.any(Array), { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + pluginIds: ['plugin-a', 'plugin-b'], + novelMappings: mappings, + }); + }); + + it('retries each downloaded-file novel when its batch restore fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 31, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + { + id: 32, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + ]; + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch failed')); + jest.mocked(_restoreNovelAndChapters).mockImplementation( + async ( + novel, + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => mappings.find(mapping => mapping.backupNovelId === novel.id)!, + ); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(1, novels[0], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(2, novels[1], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it.each([ + { + label: 'v1', + manifest: { version: '1.0.0' }, + formatVersion: 1, + }, + { + label: 'v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 2, + }, + { + label: 'v3', + manifest: { + appVersion: '3.0.0', + formatVersion: 3, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 3, + }, + ])( + '$label object payloads remain restorable', + async ({ manifest, formatVersion }) => { + const novel = makeTestNovel(301, 'legacy-source'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify(manifest); + } + return JSON.stringify(novel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 301, + pluginId: 'legacy-source', + chapters: [ + expect.objectContaining({ + id: 30101, + novelId: 301, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(result.manifest).toMatchObject({ formatVersion }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + novelMappings: [ + expect.objectContaining({ + backupNovelId: 301, + pluginId: 'legacy-source', + }), + ], + }); + }, + ); + + it('normalizes legacy object payloads before restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + chapters: [ + { + id: 50101, + novelId: 999, + path: '/legacy/1', + name: 'Legacy chapter', + }, + ], + ignoredLegacyField: 'ignored', + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 50101, + novelId: 501, + path: '/legacy/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + }); + }); + it('restores legacy novels whose stored name is empty', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + chapters: [], + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'empty-name.json', + path: '/cache/NovelAndChapters/empty-name.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + }), + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 0, + }); + }); + + it('rejects duplicate source IDs, novel identities, and chapter identities before restoring invalid files', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(601, 'source'); + const duplicateIdNovel = makeTestNovel(601, 'other-source'); + const duplicateIdentityNovel = { + ...makeTestNovel(602, 'source'), + path: validNovel.path, + }; + const duplicateChapterIdentityNovel = { + ...makeTestNovel(603, 'source'), + chapters: [ + makeTestChapter(603, 1), + { ...makeTestChapter(603, 1), id: 60302 }, + ], + }; + const payloads = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch([validNovel])), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([duplicateIdNovel])), + 'batch-000003.json': JSON.stringify( + encodeNovelBatch([duplicateIdentityNovel]), + ), + 'batch-000004.json': JSON.stringify( + encodeNovelBatch([duplicateChapterIdentityNovel]), + ), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(payloads) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return payloads[path.split('/').pop() as keyof typeof payloads]; + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith([validNovel], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 3, + pluginIds: ['source'], + }); + }); + + it('skips malformed compact files without partially restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(901, 'valid-source'); + const [validRecord] = encodeNovelBatch([validNovel]); + const malformedFiles: Record = { + 'batch-000001.json': '{', + 'batch-000002.json': JSON.stringify({}), + 'batch-000003.json': JSON.stringify([{ ...validRecord, n: undefined }]), + 'batch-000004.json': JSON.stringify([ + validRecord, + { + ...validRecord, + c: [[...validRecord.c[0]].slice(0, 14)], + }, + ]), + 'batch-000005.json': JSON.stringify([{ ...validRecord, id: 'bad' }]), + 'batch-000006.json': JSON.stringify([{ ...validRecord, p: 123 }]), + 'batch-000007.json': JSON.stringify([validRecord]), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(malformedFiles) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return malformedFiles[path.split('/').pop() ?? '']; + }); + + const result = await restoreData('/cache'); + expect( + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')), + ).toEqual( + Object.keys(malformedFiles) + .sort() + .map(name => `/cache/NovelAndChapters/${name}`), + ); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 901, + pluginId: 'valid-source', + chapters: [ + expect.objectContaining({ + id: 90101, + novelId: 901, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 7, + }); + }); + + it('treats backups without a section manifest as legacy full backups', async () => { + jest + .mocked(NativeFile.readFile) + .mockResolvedValueOnce(JSON.stringify({ version: '2.0.0' })); + + const result = await restoreData('/cache'); + + expect(result.manifest).toMatchObject({ + appVersion: '2.0.0', + formatVersion: 1, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + it('does not reject when restore mapping cleanup fails', async () => { + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + clearRestoreChapterMappingsSafely('restore-run'), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith('restore-run'); + }); +}); diff --git a/src/services/backup/restore/__tests__/local.test.ts b/src/services/backup/restore/__tests__/local.test.ts new file mode 100644 index 0000000000..6f4eac785d --- /dev/null +++ b/src/services/backup/restore/__tests__/local.test.ts @@ -0,0 +1,536 @@ +import NativeFile from '@modules/native-file'; +import NativeZipArchive from '@modules/native-zip-archive'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { restoreBackup } from '../local'; +import * as fileSections from '../files'; +import { finalizeRestoredPlugins, getRestoreCompletionText } from '../result'; +import { restoreData } from '../index'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import type { UnzipStats } from '@modules/native-zip-archive/src/NativeZipArchiveModule'; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta }; +}; + +const unzipStats: UnzipStats = { + fileCount: 7, + compressedBytes: 420, + uncompressedBytes: 840, + largestUncompressedEntryBytes: 300, + elapsedMs: 1.5, + categories: { + NovelAndChapters: { + fileCount: 2, + compressedBytes: 120, + uncompressedBytes: 240, + }, + Covers: { fileCount: 1, compressedBytes: 30, uncompressedBytes: 60 }, + NovelFiles: { fileCount: 3, compressedBytes: 240, uncompressedBytes: 480 }, + other: { fileCount: 1, compressedBytes: 30, uncompressedBytes: 60 }, + }, +}; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(), + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('../files', () => { + const actual = jest.requireActual('../files'); + return { + ...actual, + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), + }; +}); + +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); + +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(async (restoreRunId: string) => { + try { + await jest + .requireMock('@database/queries/NovelRestoreQueries') + .clearRestoreChapterMappings(restoreRunId); + } catch { + // Match the production helper's best-effort cleanup. + } + }), + restoreData: jest.fn(), +})); + +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(), +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', + ROOT_STORAGE: '/storage', +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); + +describe('local backup restore', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + beforeEach(() => { + jest.mocked(restoreData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.unzip) + .mockReset() + .mockResolvedValue(unzipStats); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.exists).mockReset().mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockReset().mockResolvedValue([]); + jest.mocked(finalizeRestoredPlugins).mockReset().mockResolvedValue([]); + jest + .mocked(clearRestoreChapterMappings) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreLegacyFiles) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreNovelFiles) + .mockReset() + .mockResolvedValue(undefined); + }); + + it('loads restored plugins after their archive is extracted', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: ['restored'], + novelMappings: [], + restoreRunId: 'restore-run-plugins', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + }, + }; + jest + .mocked(restoreData) + .mockImplementationOnce(async (_cacheDirPath, reporter) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 0.25, 'novels', true); + reporter?.('novels', 0.75, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); + return restoreResult; + }); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + const capture = createProgressCapture(); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + await restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); + expect(logSpy).toHaveBeenCalledWith( + expect.stringContaining( + `local:outer-unzip:done ${JSON.stringify(unzipStats)}`, + ), + ); + expect( + capture.progressValues.some(value => Math.abs(value - 0.09) < 0.000001), + ).toBe(true); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + const novelProgress = capture.progressValues.filter( + value => value > 0.302 && value < 0.982, + ); + expect(new Set(novelProgress).size).toBeGreaterThan(1); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + logSpy.mockRestore(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], + ); + }); + it('keeps a successful restore successful when mapping cleanup fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-cleanup-failure', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('preserves the restore error when mapping cleanup also fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-restore-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v1 downloaded archive into the legacy staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v1', + manifest: { + appVersion: '1.0.0', + formatVersion: 1 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/download.zip', + '/cache/BackupData/RestoredLegacyFiles', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredLegacyFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v2 novel-files archive into the novel staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + '/cache/BackupData/RestoredNovelFiles', + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredNovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('restores v3 novel files from the outer archive without nested extraction', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + novelDataFormat: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/NovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + expect.any(Function), + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('rejects a v3 downloaded-file restore without NovelFiles', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3-missing', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(unzipStats); + const capture = createProgressCapture(); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(capture.progressValues.every(value => value < 1)).toBe(true); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('does not report completion when plugin finalization fails', async () => { + const restoreResult = { + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-finalization-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + const failure = new Error('plugin finalization failed'); + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(finalizeRestoredPlugins).mockRejectedValueOnce(failure); + const capture = createProgressCapture(); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, capture.setMeta), + ).rejects.toBe(failure); + + expect(capture.progressValues.every(value => value < 1)).toBe(true); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + + it('does not report success when the restore pipeline fails and resets benchmark timing before a retry', async () => { + const interruption = new Error('restore interrupted'); + let completionUpdateCount = 0; + const setMeta: TaskProgressUpdater = transform => { + const next = transform({ + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }); + if (next.completionText) { + completionUpdateCount++; + } + }; + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + for (let attempt = 0; attempt < 2; attempt++) { + jest.mocked(restoreData).mockRejectedValueOnce(interruption); + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }, setMeta), + ).rejects.toBe(interruption); + } + + const startEvents = logSpy.mock.calls + .map(([message]) => String(message)) + .filter(message => message.includes('local:start')); + expect(startEvents).toHaveLength(2); + expect(startEvents[1]).not.toContain('duration:'); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + expect(completionUpdateCount).toBe(0); + logSpy.mockRestore(); + }); +}); diff --git a/src/services/backup/restore/__tests__/pipeline.test.ts b/src/services/backup/restore/__tests__/pipeline.test.ts new file mode 100644 index 0000000000..e3c56f8b20 --- /dev/null +++ b/src/services/backup/restore/__tests__/pipeline.test.ts @@ -0,0 +1,749 @@ +import { + _restoreNovelAndChapters, + _restoreNovelsAndChapters, + clearRestoreChapterMappings, +} from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; +import NativeFile from '@modules/native-file'; +import { restoreData } from '../index'; +import { restoreNovels } from '../novels'; +import { encodeNovelBatch } from '../../novelPayload'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { createRestoreProgressReporter } from '../progress'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), + clearRestoreChapterMappings: jest.fn(async () => undefined), +})); +jest.mock('@database/queries/CategoryQueries', () => ({ + _restoreCategory: jest.fn(), +})); + +jest.mock('@i18n/translations', () => ({ + getString: (key: string, options?: Record) => { + if (options?.current === undefined) { + return key; + } + return options.total === undefined + ? `${key}:${options.current}` + : `${key}:${options.current}/${options.total}`; + }, +})); + +jest.mock('@utils/mmkv/mmkv', () => ({ + MMKVStorage: { + getAllKeys: jest.fn(() => []), + getBoolean: jest.fn(), + getString: jest.fn(), + set: jest.fn(), + }, +})); + +jest.mock('@plugins/pluginManager', () => ({ + INSTALLED_PLUGINS_KEY: 'INSTALL_PLUGINS', +})); + +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + ROOT_STORAGE: '/storage', +})); + +const options = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, +}; + +const makeTestChapter = (novelId: number): ChapterInfo => ({ + id: novelId * 100 + 1, + novelId, + path: `/novel/${novelId}/chapter/1`, + name: `Chapter ${novelId}`, + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, +}); + +const makeTestNovel = (id: number, path = `/novel/${id}`): BackupNovel => ({ + id, + name: `Novel ${id}`, + path, + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: true, + isLocal: false, + totalPages: null, + chapters: [makeTestChapter(id)], +}); + +const makeMappings = (novels: BackupNovel[]): RestoredNovelMapping[] => + novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 10_000, + })); + +const createDeferred = () => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const manifestContent = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, +}); + +const configureFiles = ( + files: Record, + manifest = manifestContent, +) => { + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => + path === '/cache/NovelAndChapters' || path === '/cache/Category.json', + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(files) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path === '/cache/Version.json') { + return manifest; + } + if (path === '/cache/Category.json') { + return JSON.stringify([ + { id: 8, name: 'Reading list', novelIds: [1, 100, 200, 201] }, + ]); + } + return files[path.split('/').pop() ?? '']; + }); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const progressTexts: string[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + if (metadata.progressText !== undefined) { + progressTexts.push(metadata.progressText); + } + }; + return { + progressValues, + progressTexts, + reporter: createRestoreProgressReporter(setMeta, 'local'), + }; +}; + +const getNovelReadPaths = () => + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')); + +describe('bounded novel restore pipeline', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('settles progress and logs zeroed metrics when the novel directory is absent', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(false); + const events: string[] = []; + const capture = createProgressCapture(); + const result = await restoreNovels( + '/cache', + { + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }, + 'restore-run', + capture.reporter, + event => { + events.push(event); + }, + ); + + expect(result.failedSectionCount).toBe(1); + expect(capture.progressTexts).toContain('backupScreen.restoringNovels'); + expect( + capture.progressValues[capture.progressValues.length - 1], + ).toBeCloseTo(0.982); + const summaryPrefix = 'restoreData:database:summary '; + const summaryEvent = events.find(event => event.startsWith(summaryPrefix)); + expect(summaryEvent).toBeDefined(); + expect(JSON.parse(summaryEvent!.slice(summaryPrefix.length))).toMatchObject( + { + uniqueInputChapterCount: 0, + novelUpsertCalls: 0, + novelIdentityLookupCalls: 0, + chapterWriteChunkCalls: 0, + chapterMappingRowsAttempted: 0, + statsRefreshCalls: 0, + novelBatchFallbacks: 0, + chapterBatchFallbacks: 0, + fallbackCauses: [], + }, + ); + }); + + it('completes the progress ranges for omitted library, settings, and plugins', async () => { + const manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + sections: { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }); + configureFiles({}, manifest); + const capture = createProgressCapture(); + + await restoreData('/cache', capture.reporter); + + expect(capture.progressTexts).toContain('backupScreen.restoringNovels'); + expect(capture.progressTexts).toContain('backupScreen.restoringCategories'); + expect(capture.progressTexts).toContain('backupScreen.restoringSettings'); + expect(capture.progressTexts).toContain('backupScreen.restoringPlugins'); + expect( + capture.progressValues[capture.progressValues.length - 1], + ).toBeLessThan(1); + }); + + it('keeps the 25-novel denominator until mappings persist', async () => { + const novels = Array.from({ length: 25 }, (_, index) => + makeTestNovel(index + 1), + ); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 25, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(novels)), + }, + v3Manifest, + ); + const mappingsResult = createDeferred(); + const mappingsStarted = createDeferred(); + jest.mocked(_restoreNovelsAndChapters).mockImplementation(() => { + mappingsStarted.resolve(undefined); + return mappingsResult.promise; + }); + const progressTexts: string[] = []; + const restorePromise = restoreData('/cache', (phase, _fraction, text) => { + if (phase === 'novels') { + progressTexts.push(text); + } + }); + await mappingsStarted.promise; + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/25', + ); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:25/25', + ); + mappingsResult.resolve(makeMappings(novels)); + await restorePromise; + + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:25/25', + ); + }); + + it('uses the v2 novel file count before any mappings persist', async () => { + const novels = [makeTestNovel(1), makeTestNovel(2), makeTestNovel(3)]; + const v2Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + configureFiles( + Object.fromEntries( + novels.map(novel => [`novel-${novel.id}.json`, JSON.stringify(novel)]), + ), + v2Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async batch => makeMappings(batch)); + const firstProgress = createDeferred(); + const progressTexts: string[] = []; + const restorePromise = restoreData('/cache', (phase, _fraction, text) => { + if (phase !== 'novels') { + return; + } + progressTexts.push(text); + if (text.startsWith('backupScreen.restoringNovelsProgress:')) { + firstProgress.resolve(text); + } + }); + + await expect(firstProgress.promise).resolves.toBe( + 'backupScreen.restoringNovelsProgress:0/3', + ); + const result = await restorePromise; + + expect(result.novelCount).toBe(3); + expect(progressTexts).toContain('backupScreen.restoringNovelsProgress:3/3'); + }); + + it('restores compact v2 batches using their total novel count', async () => { + const firstBatch = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const lastNovel = makeTestNovel(101); + const v2Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstBatch)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([lastNovel])), + }, + v2Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async batch => makeMappings(batch)); + const capture = createProgressCapture(); + + const result = await restoreData('/cache', capture.reporter); + + expect(result.novelCount).toBe(101); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:101/101', + ); + }); + + it('restores successful novels after a failed three-novel batch', async () => { + const novels = Array.from({ length: 3 }, (_, index) => + makeTestNovel(index + 1), + ); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 3, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(novels)), + }, + v3Manifest, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch restore failed')); + jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => { + if (novel.id === 2) { + throw new Error('single novel restore failed'); + } + return makeMappings([novel])[0]; + }); + const capture = createProgressCapture(); + + const result = await restoreData('/cache', capture.reporter); + + expect(result.novelMappings).toEqual([ + makeMappings([novels[0]])[0], + makeMappings([novels[2]])[0], + ]); + expect(result).toMatchObject({ novelCount: 2, failedNovelCount: 1 }); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:2/3', + ); + expect( + capture.progressValues.every( + (value, index, values) => index === 0 || value >= values[index - 1], + ), + ).toBe(true); + expect(capture.progressValues.every(value => value < 1)).toBe(true); + }); + + it('validates one later file during a 100-novel write', async () => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const secondFileNovel = makeTestNovel(200); + const duplicateNovel = makeTestNovel(201, secondFileNovel.path); + const files = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([secondFileNovel])), + 'batch-000003.json': JSON.stringify(encodeNovelBatch([duplicateNovel])), + }; + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles(files, v3Manifest); + const firstWrite = createDeferred(); + const secondFileRead = createDeferred(); + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + let writeCount = 0; + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async (novels, restoreOptions) => { + writeCount++; + restoreOptions?.onProgress?.({ + stage: 'chapters', + completed: 0, + total: novels.length, + }); + if (writeCount === 1) { + return firstWrite.promise; + } + return makeMappings(novels); + }); + const capture = createProgressCapture(); + + const restorePromise = restoreData('/cache', capture.reporter); + await secondFileRead.promise; + + expect(getNovelReadPaths()).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenNthCalledWith( + 1, + firstFileNovels, + expect.objectContaining({ + includeChapterMappings: true, + restoreRunId: expect.any(String), + }), + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(capture.progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + + firstWrite.resolve(makeMappings(firstFileNovels)); + const result = await restorePromise; + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:101/101', + ); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(2); + expect(_restoreNovelsAndChapters).toHaveBeenNthCalledWith( + 2, + [secondFileNovel], + expect.objectContaining({ + includeChapterMappings: true, + restoreRunId: result.restoreRunId, + }), + ); + expect(getNovelReadPaths()).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + '/cache/NovelAndChapters/batch-000003.json', + ]); + expect(result).toMatchObject({ + novelCount: 101, + categoryCount: 1, + failedNovelCount: 1, + failedCategoryCount: 0, + pluginIds: ['source'], + novelMappings: [ + ...makeMappings(firstFileNovels), + ...makeMappings([secondFileNovel]), + ], + }); + const [restoredCategory, restoredNovelIdMap] = ( + _restoreCategory as jest.Mock + ).mock.calls[0]; + expect(restoredCategory).toEqual({ + id: 8, + name: 'Reading list', + novelIds: [1, 100, 200], + }); + expect(restoredNovelIdMap.get(1)).toBe(10_001); + expect(restoredNovelIdMap.get(100)).toBe(10_100); + expect(restoredNovelIdMap.get(200)).toBe(10_200); + }); + it('reports completion for empty and invalid files without novel writes', async () => { + configureFiles({ + 'empty.json': JSON.stringify(encodeNovelBatch([])), + 'invalid.json': '{', + }); + const capture = createProgressCapture(); + const result = await restoreNovels( + '/cache', + { + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }, + 'restore-run', + capture.reporter, + ); + + expect(_restoreNovelsAndChapters).not.toHaveBeenCalled(); + expect(capture.progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/1', + ); + expect(result).toMatchObject({ novelCount: 0, failedNovelCount: 1 }); + }); + + it('settles the active write when the progress updater throws undefined', async () => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const nextFileNovel = makeTestNovel(200); + const v3Manifest = JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + novelDataFormat: 2, + novelCount: 101, + sections: options, + }); + configureFiles( + { + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([nextFileNovel])), + }, + v3Manifest, + ); + const firstWrite = createDeferred(); + let firstWriteSettled = false; + const writePromise = firstWrite.promise.then(mappings => { + firstWriteSettled = true; + return mappings; + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async (_batch, restoreOptions) => { + restoreOptions?.onProgress?.({ + stage: 'chapters', + completed: 0, + total: 1, + }); + return writePromise; + }); + const secondFileRead = createDeferred(); + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + const interruption = undefined; + const progressTexts: string[] = []; + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progressText) { + progressTexts.push(metadata.progressText); + } + if (metadata.progress !== undefined && metadata.progress > 0.4) { + throw interruption; + } + }; + + const restorePromise = restoreData( + '/cache', + createRestoreProgressReporter(setMeta, 'local'), + ); + await secondFileRead.promise; + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).not.toHaveBeenCalled(); + expect(progressTexts).toContain( + 'backupScreen.restoringNovelsProgress:0/101', + ); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + + firstWrite.resolve(makeMappings(firstFileNovels)); + await expect(restorePromise).rejects.toBe(interruption); + expect(progressTexts).not.toContain( + 'backupScreen.restoringNovelsProgress:100/101', + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + + expect(firstWriteSettled).toBe(true); + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(clearRestoreChapterMappings).toHaveBeenCalledTimes(1); + }); + + it.each([ + { label: 'Error', interruption: new Error('progress updater failed') }, + { label: 'undefined', interruption: undefined }, + ])( + 'does not start fallback after prefetched progress throws $label', + async ({ interruption }) => { + const firstFileNovels = Array.from({ length: 100 }, (_, index) => + makeTestNovel(index + 1), + ); + const nextFileNovel = makeTestNovel(200); + configureFiles({ + 'batch-000001.json': JSON.stringify(encodeNovelBatch(firstFileNovels)), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([nextFileNovel])), + }); + + const firstWrite = createDeferred(); + const databaseError = new Error('database batch failed'); + let firstWriteSettled = false; + const writePromise = firstWrite.promise.then( + mappings => { + firstWriteSettled = true; + return mappings; + }, + error => { + firstWriteSettled = true; + throw error; + }, + ); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(() => writePromise); + jest + .mocked(_restoreNovelAndChapters) + .mockImplementation(async novel => makeMappings([novel])[0]); + + const secondFileRead = createDeferred(); + const updaterInterrupted = createDeferred(); + let currentTime = 1_000; + const dateNow = jest + .spyOn(Date, 'now') + .mockImplementation(() => currentTime); + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }; + let interruptionThrown = false; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if ( + !interruptionThrown && + metadata.progress !== undefined && + metadata.progress > 0.4 + ) { + interruptionThrown = true; + updaterInterrupted.resolve(undefined); + throw interruption; + } + }; + const readFile = jest.mocked(NativeFile.readFile); + const readFileImplementation = readFile.getMockImplementation()!; + readFile.mockImplementation(async path => { + if (path.endsWith('batch-000002.json')) { + currentTime += 250; + secondFileRead.resolve(undefined); + } + return readFileImplementation(path); + }); + jest.mocked(clearRestoreChapterMappings).mockImplementation(async () => { + expect(firstWriteSettled).toBe(true); + }); + + try { + const restorePromise = restoreData( + '/cache', + createRestoreProgressReporter(setMeta, 'local'), + ); + await secondFileRead.promise; + await updaterInterrupted.promise; + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).not.toHaveBeenCalled(); + + firstWrite.reject(databaseError); + await expect(restorePromise).rejects.toBe(interruption); + + expect(firstWriteSettled).toBe(true); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledTimes(1); + } finally { + dateNow.mockRestore(); + } + }, + ); +}); diff --git a/src/services/backup/restore/__tests__/progress.test.ts b/src/services/backup/restore/__tests__/progress.test.ts new file mode 100644 index 0000000000..704f169071 --- /dev/null +++ b/src/services/backup/restore/__tests__/progress.test.ts @@ -0,0 +1,149 @@ +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { createRestoreProgressReporter } from '../progress'; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + completionText: 'preserved', + }; + const updates: BackgroundTaskMetadata[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + updates.push(metadata); + }; + return { setMeta, updates }; +}; + +describe('restore progress reporter', () => { + afterEach(() => { + jest.useRealTimers(); + }); + it('maps local phases into benchmark-weighted ranges and reaches one at finalize', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', 1, 'copy'); + reporter('extract', 1, 'extract'); + reporter('manifest', 1, 'manifest'); + reporter('novels', 1, 'novels'); + reporter('categories', 1, 'categories'); + reporter('settings', 1, 'settings'); + reporter('plugins', 1, 'plugins'); + reporter('selectedFiles', 1, 'selected files'); + reporter('finalize', 0, 'finishing'); + reporter('finalize', 1, 'done'); + + const values = updates.map(update => update.progress!); + expect(values[0]).toBeCloseTo(0.09); + expect(values[1]).toBeCloseTo(0.3); + expect(values[2]).toBeCloseTo(0.302); + expect(values[3]).toBeCloseTo(0.982); + expect(values[4]).toBeCloseTo(0.986); + expect(values[5]).toBeCloseTo(0.988); + expect(values[6]).toBeCloseTo(0.992); + expect(values[7]).toBeCloseTo(0.997); + expect(values[8]).toBeCloseTo(0.997); + expect(values[9]).toBe(1); + expect( + values.every((value, index) => index === 0 || value >= values[index - 1]), + ).toBe(true); + expect(updates.every(update => update.completionText === 'preserved')).toBe( + true, + ); + }); + + it('combines remote retrieval and extraction in the 0.30 source range', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'remote')!; + + reporter('source', 1, 'download'); + reporter('extract', 1, 'download'); + reporter('manifest', 1, 'manifest'); + + expect(updates[0].progress).toBeCloseTo(0.3); + expect(updates[updates.length - 1].progress).toBeCloseTo(0.302); + }); + it.each(['local', 'remote'] as const)( + 'reserves completion for finalization fraction one for %s restores', + source => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, source)!; + + for (const fraction of [0, 0.25, 0.75, 1 - Number.EPSILON]) { + reporter('finalize', fraction, `finishing ${fraction}`, true); + } + + const partialValues = updates.map(update => update.progress!); + expect(partialValues.every(value => value < 1)).toBe(true); + expect( + partialValues.every( + (value, index) => index === 0 || value >= partialValues[index - 1], + ), + ).toBe(true); + + reporter('finalize', 1, 'done', true); + expect(updates[updates.length - 1].progress).toBe(1); + }, + ); + + it('clamps fractions and keeps progress monotonic across interleaved phases', () => { + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', -4, 'start'); + reporter('extract', 2, 'extracted'); + reporter('source', 0, 'interleaved source'); + reporter('manifest', Number.NaN, 'manifest'); + + const values = updates.map(update => update.progress!); + expect(values[0]).toBe(0); + expect(values[1]).toBeCloseTo(0.3); + expect(values[2]).toBeCloseTo(0.3); + expect(values[3]).toBeCloseTo(0.3); + expect(values.every(value => value >= 0 && value <= 1)).toBe(true); + expect( + values.every((value, index) => index === 0 || value >= values[index - 1]), + ).toBe(true); + }); + + it('throttles ordinary updates but emits phase changes, forced updates, and completion', () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const { setMeta, updates } = createProgressCapture(); + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + reporter('source', 0, 'source'); + reporter('source', 0.2, 'ordinary'); + expect(updates).toHaveLength(1); + + jest.advanceTimersByTime(250); + reporter('source', 0.2, 'ordinary'); + expect(updates).toHaveLength(2); + + reporter('source', 0.3, 'forced', true); + reporter('source', 0.3, 'forced', true); + expect(updates).toHaveLength(3); + + reporter('extract', 0, 'next phase'); + expect(updates).toHaveLength(4); + reporter('extract', 1, 'next phase complete'); + expect(updates).toHaveLength(5); + }); + + it('does not swallow updater exceptions', () => { + const setMeta: TaskProgressUpdater = () => { + throw new Error('metadata update failed'); + }; + const reporter = createRestoreProgressReporter(setMeta, 'local')!; + + expect(() => reporter('source', 0, 'copy')).toThrow( + 'metadata update failed', + ); + }); +}); diff --git a/src/services/backup/__tests__/restoreResult.test.ts b/src/services/backup/restore/__tests__/result.test.ts similarity index 98% rename from src/services/backup/__tests__/restoreResult.test.ts rename to src/services/backup/restore/__tests__/result.test.ts index 74932e74cc..7b14243602 100644 --- a/src/services/backup/__tests__/restoreResult.test.ts +++ b/src/services/backup/restore/__tests__/result.test.ts @@ -5,7 +5,7 @@ import { getMissingRestorePluginIds, getRestoreCompletionText, type RestoreResult, -} from '../restoreResult'; +} from '../result'; jest.mock('@i18n/translations', () => ({ getString: (key: string, options?: Record) => { @@ -65,6 +65,7 @@ const successfulResult: RestoreResult = { failedSectionCount: 0, pluginIds: ['installed'], novelMappings: [], + restoreRunId: 'restore-run-1', manifest: { appVersion: '2.1.0', formatVersion: 2, diff --git a/src/services/backup/restore/__tests__/selfhost.test.ts b/src/services/backup/restore/__tests__/selfhost.test.ts new file mode 100644 index 0000000000..44b35f745d --- /dev/null +++ b/src/services/backup/restore/__tests__/selfhost.test.ts @@ -0,0 +1,258 @@ +import { download } from '@api/remote'; +import type { + BackgroundTaskMetadata, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { CACHE_DIR_PATH, clearBackupCache } from '../../cache'; +import { ZipBackupName } from '../../types'; +import { restoreLegacyFiles, restoreNovelFiles } from '../files'; +import { selfHostRestore } from '../selfhost'; + +import { restoreData } from '../index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from '../result'; +import type { RestoreProgressReporter } from '../progress'; + +jest.mock('@api/remote', () => ({ download: jest.fn() })); +jest.mock('../../cache', () => ({ + CACHE_DIR_PATH: '/cache/BackupData', + clearBackupCache: jest.fn(), +})); +jest.mock('../files', () => ({ + getLegacyFilesRestorePath: jest.fn(() => '/cache/legacy-files'), + getNovelFilesRestorePath: jest.fn(() => '/cache/novel-files'), + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), +})); +jest.mock('../index', () => ({ + clearRestoreChapterMappingsSafely: jest.fn(), + restoreData: jest.fn(), +})); +jest.mock('../result', () => ({ + finalizeRestoredPlugins: jest.fn(), + getRestoreCompletionText: jest.fn(() => 'completed'), +})); +jest.mock('@i18n/translations', () => ({ + getString: (key: string) => key, +})); +jest.mock('@utils/Storages', () => ({ + NOVEL_STORAGE: '/storage/Novels', + PLUGIN_STORAGE: '/storage/Plugins', +})); + +const backupFolder = 'backup-folder'; +const host = 'https://example.invalid'; +const restoreDataFor = ( + plugins: boolean, + downloadedFiles = false, + formatVersion: 1 | 2 | 3 = 3, + novelMappings: RestoreResult['novelMappings'] = [], +): RestoreResult => ({ + novelCount: 0, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: plugins ? ['restored-plugin'] : [], + novelMappings, + restoreRunId: 'selfhost-restore-run', + manifest: { + appVersion: '2.1.3', + formatVersion, + novelDataFormat: 2, + sections: { + library: downloadedFiles, + settings: false, + plugins, + downloadedFiles, + }, + }, +}); + +const reportRestoreStages = (reporter: RestoreProgressReporter | undefined) => { + reporter?.('manifest', 1, 'manifest', true); + reporter?.('novels', 1, 'novels', true); + reporter?.('categories', 1, 'categories', true); + reporter?.('settings', 1, 'settings', true); + reporter?.('plugins', 1, 'plugins', true); +}; + +const createProgressCapture = () => { + let metadata: BackgroundTaskMetadata = { + name: 'SELF_HOST_RESTORE', + isRunning: false, + progress: undefined, + progressText: undefined, + }; + const progressValues: number[] = []; + const setMeta: TaskProgressUpdater = transform => { + metadata = transform(metadata); + if (metadata.progress !== undefined) { + progressValues.push(metadata.progress); + } + }; + return { progressValues, setMeta, getMetadata: () => metadata }; +}; + +describe('self-host restore progress', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(download).mockResolvedValue(''); + jest.mocked(clearBackupCache).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValue([]); + }); + + it('keeps remote retrieval and later phases monotonic through completion', async () => { + const result = restoreDataFor(false); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(capture.progressValues[0]).toBe(0); + expect( + capture.progressValues.some(value => Math.abs(value - 0.3) < 0.000001), + ).toBe(true); + expect(capture.progressValues[capture.progressValues.length - 1]).toBe(1); + expect( + capture.progressValues.every( + (value, index) => + index === 0 || value >= capture.progressValues[index - 1], + ), + ).toBe(true); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.DATA, + CACHE_DIR_PATH, + ); + }); + + it('does not report completion when a selected archive download fails', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(true, true, 2, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + jest.mocked(download).mockImplementation(async (_host, _folder, name) => { + if (name === ZipBackupName.NOVEL_FILES) { + throw new Error('selected archive download failed'); + } + return ''; + }); + const capture = createProgressCapture(); + + await expect( + selfHostRestore({ host, backupFolder }, capture.setMeta), + ).rejects.toThrow('selected archive download failed'); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.NOVEL_FILES, + '/cache/novel-files', + ); + expect(restoreNovelFiles).not.toHaveBeenCalled(); + expect(capture.progressValues.length).toBeGreaterThan(0); + expect(Math.max(...capture.progressValues)).toBeLessThan(1); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(getRestoreCompletionText).not.toHaveBeenCalled(); + }); + + it('restores selected format-2 archives before finalizing plugins', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(true, true, 2, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.PLUGINS, + '/storage/Plugins', + ); + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.NOVEL_FILES, + '/cache/novel-files', + ); + expect(restoreNovelFiles).toHaveBeenCalledWith( + '/cache/novel-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect(finalizeRestoredPlugins).toHaveBeenCalledWith(result); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + Math.max(...jest.mocked(download).mock.invocationCallOrder), + ); + expect( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(restoreNovelFiles).mock.invocationCallOrder[0], + ); + expect( + jest.mocked(getRestoreCompletionText).mock.invocationCallOrder[0], + ).toBeGreaterThan( + jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], + ); + expect(capture.getMetadata().isRunning).toBe(false); + expect(capture.getMetadata().completionText).toBe('completed'); + }); + + it('restores format-1 downloads from the legacy staging path', async () => { + const mappings = [ + { pluginId: 'plugin-a', backupNovelId: 12, restoredNovelId: 42 }, + ]; + const result = restoreDataFor(false, true, 1, mappings); + jest.mocked(restoreData).mockImplementationOnce(async (_path, reporter) => { + reportRestoreStages(reporter); + return result; + }); + const capture = createProgressCapture(); + + await selfHostRestore({ host, backupFolder }, capture.setMeta); + + expect(download).toHaveBeenCalledWith( + host, + backupFolder, + ZipBackupName.DOWNLOAD, + '/cache/legacy-files', + ); + expect(restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/legacy-files', + mappings, + result.restoreRunId, + expect.any(Function), + ); + expect(restoreNovelFiles).not.toHaveBeenCalled(); + expect(capture.getMetadata().completionText).toBe('completed'); + }); +}); diff --git a/src/services/backup/restore/__tests__/validation.test.ts b/src/services/backup/restore/__tests__/validation.test.ts new file mode 100644 index 0000000000..01ab70d641 --- /dev/null +++ b/src/services/backup/restore/__tests__/validation.test.ts @@ -0,0 +1,763 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; +import { decodeNovelBatch, encodeNovelBatch } from '../../novelPayload'; +import type { ResolvedBackupManifest } from '../../types'; +import { + decodeAndValidateNovelFile, + decodeRestoreNovelFile, + getNovelFileRecordCount, + NovelFileValidationError, +} from '../validation'; + +jest.mock('@utils/Storages', () => ({ ROOT_STORAGE: '/storage' })); + +const sections = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, +}; + +const makeManifest = (novelDataFormat: 1 | 2): ResolvedBackupManifest => ({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat, + sections, +}); + +const legacyManifest = makeManifest(1); +const compactManifest = makeManifest(2); + +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: `2024-03-${String(chapterNumber).padStart(2, '0')}`, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); + +type MutableLegacyNovel = Record & { + chapters: Record[]; +}; + +type MutableCompactNovel = Record & { c: unknown[][] }; + +type SeenRecords = { + novelIds: Set; + novelIdentities: Set; + chapterIdentities: Set; +}; + +const newSeenRecords = (): SeenRecords => ({ + novelIds: new Set(), + novelIdentities: new Set(), + chapterIdentities: new Set(), +}); + +const makeLegacyPayload = (): MutableLegacyNovel => ({ + id: 1, + path: '/legacy/one', + pluginId: 'legacy-source', + name: 'Legacy novel', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 101, + novelId: 1, + path: '/legacy/one/chapter/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], +}); + +const makeCompactRecord = ( + novel: BackupNovel = makeTestNovel(1), +): MutableCompactNovel => + encodeNovelBatch([novel])[0] as unknown as MutableCompactNovel; + +const setLegacyNovelField = (field: string, value: unknown) => { + const payload = makeLegacyPayload(); + payload[field] = value; + return payload; +}; + +const setLegacyChapterField = (field: string, value: unknown) => { + const payload = makeLegacyPayload(); + const chapter = payload.chapters[0]; + if (!chapter) { + throw new Error('Expected a legacy chapter fixture'); + } + chapter[field] = value; + return payload; +}; + +const setCompactNovelField = (field: string, value: unknown) => { + const record = makeCompactRecord(); + record[field] = value; + return [record]; +}; + +const setCompactChapterField = (index: number, value: unknown) => { + const record = makeCompactRecord(); + const chapter = record.c[0]; + if (!chapter) { + throw new Error('Expected a compact chapter fixture'); + } + chapter[index] = value; + return [record]; +}; + +const encodeFile = (novels: BackupNovel[]) => + JSON.stringify(encodeNovelBatch(novels)); + +const captureValidationError = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seen: SeenRecords = newSeenRecords(), +): NovelFileValidationError => { + try { + decodeAndValidateNovelFile( + fileContent, + manifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ); + } catch (error) { + expect(error).toBeInstanceOf(NovelFileValidationError); + return error as NovelFileValidationError; + } + throw new Error('Expected novel file validation to fail'); +}; + +describe('decodeRestoreNovelFile', () => { + it('normalizes legacy values and ignores fields outside the backup shape', () => { + const payload = { + id: 22, + path: '/legacy/empty-name', + pluginId: 'legacy-source', + name: '', + cover: '/covers/legacy.jpg', + chapters: [ + { + id: 2201, + novelId: 999, + path: '/legacy/empty-name/chapter/1', + name: '', + extraChapterField: 'discard me', + }, + ], + extraNovelField: 'discard me', + }; + + expect(decodeRestoreNovelFile(payload, legacyManifest)).toEqual([ + { + id: 22, + path: '/legacy/empty-name', + pluginId: 'legacy-source', + name: '', + cover: 'file:///storage/covers/legacy.jpg', + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 2201, + novelId: 22, + path: '/legacy/empty-name/chapter/1', + name: '', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ]); + }); + + it('maps every compact chapter tuple field and removes restore-only aggregates', () => { + const chapter: ChapterInfo = { + id: 1402, + novelId: 14, + path: '/novel/14/chapter/2', + name: 'Mapped chapter', + releaseTime: 'released-at', + bookmark: true, + unread: false, + readTime: 'read-at', + isDownloaded: false, + updatedTime: 'updated-at', + chapterNumber: 2.5, + page: 'page-token', + position: 3.5, + progress: 0.625, + scanlator: 'scanlator-team', + timeSpent: 91, + }; + const novel: BackupNovel = { + ...makeTestNovel(14, 'compact-source', [chapter]), + name: 'Compact mapping novel', + path: '/novel/14', + cover: '/covers/round-trip.jpg', + }; + const novelWithAggregates = { + ...novel, + chaptersDownloaded: 4, + chaptersUnread: 6, + totalChapters: 12, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + }; + const compactPayload = encodeNovelBatch([novelWithAggregates]); + + expect(compactPayload[0]?.c).toEqual([ + [ + 1402, + '/novel/14/chapter/2', + 'Mapped chapter', + 'released-at', + true, + false, + 'read-at', + false, + 'updated-at', + 2.5, + 'page-token', + 3.5, + 0.625, + 'scanlator-team', + 91, + ], + ]); + expect(decodeNovelBatch(compactPayload)).toEqual([novelWithAggregates]); + + expect(decodeRestoreNovelFile(compactPayload, compactManifest)).toEqual([ + { + ...novel, + cover: 'file:///storage/covers/round-trip.jpg', + }, + ]); + }); + + it.each([ + ['HTTP', 'https://cdn.example/cover.jpg', 'https://cdn.example/cover.jpg'], + [ + '/covers/relative.jpg', + '/covers/relative.jpg', + 'file:///storage/covers/relative.jpg', + ], + ['empty', '', ''], + ['null', null, null], + ] as const)( + 'keeps the existing %s compact cover handling', + (_description, cover, expectedCover) => { + const novel = { ...makeTestNovel(40), cover }; + const [restored] = decodeRestoreNovelFile( + encodeNovelBatch([novel]), + compactManifest, + ); + + expect(restored?.cover).toBe(expectedCover); + }, + ); +}); + +type InvalidKind = 'string' | 'boolean' | 'number'; +type MalformedInputCase = { + label: string; + manifest: ResolvedBackupManifest; + payload: () => unknown; +}; + +const invalidValueForKind: Record = { + string: 42, + boolean: 'not-a-boolean', + number: 'not-a-number', +}; + +const nullableFieldCases = ( + prefix: string, + manifest: ResolvedBackupManifest, + fields: readonly (readonly [string, InvalidKind])[], + makePayload: (field: string, value: unknown) => unknown, +): MalformedInputCase[] => + fields.map(([field, kind]) => ({ + label: `${prefix} ${field}`, + manifest, + payload: () => makePayload(field, invalidValueForKind[kind]), + })); + +const compactChapterNullableFields: readonly (readonly [ + string, + number, + InvalidKind, +])[] = [ + ['releaseTime', 3, 'string'], + ['bookmark', 4, 'boolean'], + ['unread', 5, 'boolean'], + ['readTime', 6, 'string'], + ['isDownloaded', 7, 'boolean'], + ['updatedTime', 8, 'string'], + ['chapterNumber', 9, 'number'], + ['page', 10, 'string'], + ['position', 11, 'number'], + ['progress', 12, 'number'], + ['scanlator', 13, 'string'], + ['timeSpent', 14, 'number'], +]; + +const malformedInputCases: MalformedInputCase[] = [ + { + label: 'legacy novel id', + manifest: legacyManifest, + payload: () => setLegacyNovelField('id', 0), + }, + { + label: 'legacy novel path', + manifest: legacyManifest, + payload: () => setLegacyNovelField('path', ''), + }, + { + label: 'legacy plugin id', + manifest: legacyManifest, + payload: () => setLegacyNovelField('pluginId', ''), + }, + { + label: 'legacy novel name type', + manifest: legacyManifest, + payload: () => setLegacyNovelField('name', null), + }, + { + label: 'legacy chapter id', + manifest: legacyManifest, + payload: () => setLegacyChapterField('id', 0), + }, + { + label: 'legacy chapter novel id', + manifest: legacyManifest, + payload: () => setLegacyChapterField('novelId', 0), + }, + { + label: 'legacy chapter path', + manifest: legacyManifest, + payload: () => setLegacyChapterField('path', ''), + }, + { + label: 'legacy chapter name type', + manifest: legacyManifest, + payload: () => setLegacyChapterField('name', false), + }, + { + label: 'legacy chapters collection', + manifest: legacyManifest, + payload: () => setLegacyNovelField('chapters', {}), + }, + ...nullableFieldCases( + 'legacy novel', + legacyManifest, + [ + ['cover', 'string'], + ['summary', 'string'], + ['author', 'string'], + ['artist', 'string'], + ['status', 'string'], + ['genres', 'string'], + ['inLibrary', 'boolean'], + ['isLocal', 'boolean'], + ['totalPages', 'number'], + ], + setLegacyNovelField, + ), + ...nullableFieldCases( + 'legacy chapter', + legacyManifest, + [ + ['releaseTime', 'string'], + ['readTime', 'string'], + ['bookmark', 'boolean'], + ['unread', 'boolean'], + ['isDownloaded', 'boolean'], + ['updatedTime', 'string'], + ['chapterNumber', 'number'], + ['page', 'string'], + ['progress', 'number'], + ['position', 'number'], + ['scanlator', 'string'], + ['timeSpent', 'number'], + ], + setLegacyChapterField, + ), + { + label: 'compact novel id', + manifest: compactManifest, + payload: () => setCompactNovelField('id', 0), + }, + { + label: 'compact novel path', + manifest: compactManifest, + payload: () => setCompactNovelField('p', ''), + }, + { + label: 'compact plugin id', + manifest: compactManifest, + payload: () => setCompactNovelField('pi', ''), + }, + { + label: 'compact novel name type', + manifest: compactManifest, + payload: () => setCompactNovelField('n', null), + }, + { + label: 'compact chapter id', + manifest: compactManifest, + payload: () => setCompactChapterField(0, 0), + }, + { + label: 'compact chapter path', + manifest: compactManifest, + payload: () => setCompactChapterField(1, ''), + }, + { + label: 'compact chapter name type', + manifest: compactManifest, + payload: () => setCompactChapterField(2, false), + }, + { + label: 'compact chapter collection', + manifest: compactManifest, + payload: () => setCompactNovelField('c', {}), + }, + ...nullableFieldCases( + 'compact novel', + compactManifest, + [ + ['co', 'string'], + ['s', 'string'], + ['a', 'string'], + ['ar', 'string'], + ['st', 'string'], + ['g', 'string'], + ['l', 'boolean'], + ['lo', 'boolean'], + ['t', 'number'], + ['d', 'number'], + ['u', 'number'], + ['tc', 'number'], + ['lr', 'string'], + ['lu', 'string'], + ], + setCompactNovelField, + ), + ...compactChapterNullableFields.map(([field, index, kind]) => ({ + label: `compact chapter ${field}`, + manifest: compactManifest, + payload: () => setCompactChapterField(index, invalidValueForKind[kind]), + })), +]; + +describe('malformed backup records', () => { + it.each(malformedInputCases)('rejects $label', ({ manifest, payload }) => { + expect(() => decodeRestoreNovelFile(payload(), manifest)).toThrow(); + }); + + it.each([ + ['missing', (record: MutableCompactNovel) => delete record.p], + ['extra', (record: MutableCompactNovel) => (record.extra = true)], + ])('rejects compact records with %s novel keys', (_shape, mutate) => { + const record = makeCompactRecord(); + mutate(record); + + expect(() => decodeNovelBatch([record])).toThrow(); + }); + + it.each([ + ['short', 14], + ['long', 16], + ] as const)('rejects a %s compact chapter tuple', (_shape, length) => { + const record = makeCompactRecord(); + const chapter = record.c[0]; + if (!chapter) { + throw new Error('Expected a compact chapter fixture'); + } + record.c[0] = + length === 14 ? chapter.slice(0, length) : [...chapter, 'extra']; + + expect(() => decodeNovelBatch([record])).toThrow( + 'Invalid compact novel record', + ); + }); + + it.each([ + ['novel id', () => setCompactNovelField('id', Number.NaN)], + [ + 'aggregate count', + () => setCompactNovelField('d', Number.POSITIVE_INFINITY), + ], + [ + 'chapter number', + () => setCompactChapterField(9, Number.NEGATIVE_INFINITY), + ], + ])('rejects non-finite %s values in the compact codec', (_field, payload) => { + expect(() => decodeNovelBatch(payload())).toThrow( + 'Invalid compact novel record', + ); + }); +}); + +describe('decodeAndValidateNovelFile', () => { + it.each([ + { + label: 'novel ID', + novels: [makeTestNovel(31), makeTestNovel(31, 'other-source')], + message: 'Duplicate backup novel identity', + }, + { + label: 'plugin and path identity', + novels: [ + makeTestNovel(32, 'source'), + { ...makeTestNovel(33, 'source'), path: '/novel/32' }, + ], + message: 'Duplicate backup novel identity', + }, + { + label: 'chapter path identity', + novels: [ + makeTestNovel(34, 'source', [ + makeTestChapter(34, 1), + { + ...makeTestChapter(34, 2), + path: makeTestChapter(34, 1).path, + }, + ]), + ], + message: 'Duplicate backup chapter identity', + }, + ])('rejects duplicate $label values within a file', ({ novels, message }) => { + expect( + captureValidationError(encodeFile(novels), compactManifest).message, + ).toBe(message); + }); + + it.each([ + { + label: 'novel ID', + candidate: (accepted: BackupNovel) => ({ + ...makeTestNovel(502, 'other-source'), + id: accepted.id, + path: '/novel/different-path', + }), + }, + { + label: 'plugin and path identity', + candidate: (accepted: BackupNovel) => ({ + ...makeTestNovel(503, accepted.pluginId), + path: accepted.path, + }), + }, + ])( + 'rejects duplicate $label identities from a prior file', + ({ candidate }) => { + const seen = newSeenRecords(); + const accepted = makeTestNovel(501, 'source'); + expect( + decodeAndValidateNovelFile( + encodeFile([accepted]), + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toHaveLength(1); + + const error = captureValidationError( + encodeFile([candidate(accepted)]), + compactManifest, + seen, + ); + expect(error.message).toBe('Duplicate backup novel identity'); + }, + ); + + it('preserves all identity sets after late failure and accepts valid records', () => { + const seen: SeenRecords = { + novelIds: new Set([900]), + novelIdentities: new Set(['prior-source\u0000/prior-novel']), + chapterIdentities: new Set(['900\u0000/prior-chapter']), + }; + const before = { + novelIds: new Set(seen.novelIds), + novelIdentities: new Set(seen.novelIdentities), + chapterIdentities: new Set(seen.chapterIdentities), + }; + const first = makeTestNovel(601); + const second = makeTestNovel(602); + const duplicateChapter = makeTestChapter(603, 1); + const lateFailure = makeTestNovel(603, 'source', [ + duplicateChapter, + { ...makeTestChapter(603, 2), path: duplicateChapter.path }, + ]); + + const error = captureValidationError( + encodeFile([first, second, lateFailure]), + compactManifest, + seen, + ); + + expect(error.recordCount).toBe(3); + expect(seen.novelIds).toEqual(before.novelIds); + expect(seen.novelIdentities).toEqual(before.novelIdentities); + expect(seen.chapterIdentities).toEqual(before.chapterIdentities); + + expect( + decodeAndValidateNovelFile( + encodeFile([first, second]), + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toHaveLength(2); + expect([...seen.novelIds]).toEqual([900, first.id, second.id]); + expect(seen.novelIdentities.size).toBe(3); + expect(seen.chapterIdentities.size).toBe(3); + }); + + it('counts a late malformed chapter without accepting earlier records', () => { + const seen = newSeenRecords(); + const first = makeTestNovel(701); + const second = makeTestNovel(702); + const last = makeCompactRecord(makeTestNovel(703)); + const lastChapter = last.c[0]; + if (!lastChapter) { + throw new Error('Expected a compact chapter fixture'); + } + lastChapter[1] = ''; + + const error = captureValidationError( + JSON.stringify([ + makeCompactRecord(first), + makeCompactRecord(second), + last, + ]), + compactManifest, + seen, + ); + + expect(error.recordCount).toBe(3); + expect(seen.novelIds.size).toBe(0); + expect(seen.novelIdentities.size).toBe(0); + expect(seen.chapterIdentities.size).toBe(0); + }); + + it('counts failures and accepts an empty compact batch', () => { + expect(captureValidationError('{', compactManifest).recordCount).toBe(1); + expect(captureValidationError('{}', compactManifest).recordCount).toBe(1); + expect( + captureValidationError(JSON.stringify({}), legacyManifest).recordCount, + ).toBe(1); + + const oversizedPayload = Array.from({ length: 101 }, () => null); + expect(getNovelFileRecordCount(oversizedPayload, compactManifest)).toBe( + 101, + ); + expect( + captureValidationError(JSON.stringify(oversizedPayload), compactManifest) + .recordCount, + ).toBe(101); + + const malformedRecord = makeCompactRecord(); + malformedRecord.n = null; + expect( + captureValidationError( + JSON.stringify([makeCompactRecord(), malformedRecord]), + compactManifest, + ).recordCount, + ).toBe(2); + + expect(getNovelFileRecordCount({}, compactManifest)).toBe(1); + expect(getNovelFileRecordCount([], compactManifest)).toBe(1); + expect(getNovelFileRecordCount([null, null], compactManifest)).toBe(2); + expect(getNovelFileRecordCount([null, null], legacyManifest)).toBe(1); + expect(getNovelFileRecordCount({}, legacyManifest)).toBe(1); + + expect(decodeRestoreNovelFile([], compactManifest)).toEqual([]); + const seen = newSeenRecords(); + expect( + decodeAndValidateNovelFile( + '[]', + compactManifest, + seen.novelIds, + seen.novelIdentities, + seen.chapterIdentities, + ), + ).toEqual([]); + expect(seen.novelIds.size).toBe(0); + expect(seen.novelIdentities.size).toBe(0); + expect(seen.chapterIdentities.size).toBe(0); + }); +}); diff --git a/src/services/backup/restore/drive.ts b/src/services/backup/restore/drive.ts new file mode 100644 index 0000000000..22f32db126 --- /dev/null +++ b/src/services/backup/restore/drive.ts @@ -0,0 +1,43 @@ +import { DriveFile } from '@api/drive/types'; +import { exists } from '@api/drive'; +import { download } from '@api/drive/request'; +import { getString } from '@i18n/translations'; +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import { ZipBackupName } from '../types'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +export const driveRestore = async ( + backupFolder: DriveFile, + setMeta: TaskProgressUpdater, +) => { + await runRestore( + { + kind: 'remote', + prepare: async report => { + const progressText = getString('backupScreen.downloadingData'); + report?.('source', 0, progressText, true); + const dataFile = await exists( + ZipBackupName.DATA, + false, + backupFolder.id, + ); + if (!dataFile) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await clearBackupCache(); + await download(dataFile, CACHE_DIR_PATH); + report?.('source', 1, progressText, true); + report?.('extract', 1, progressText, true); + }, + loadArchive: async (name, destination) => { + const archive = await exists(name, false, backupFolder.id); + if (!archive) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await download(archive, destination); + }, + }, + setMeta, + ); +}; diff --git a/src/services/backup/restore/files.ts b/src/services/backup/restore/files.ts new file mode 100644 index 0000000000..495849f853 --- /dev/null +++ b/src/services/backup/restore/files.ts @@ -0,0 +1,116 @@ +import { NOVEL_STORAGE, PLUGIN_STORAGE } from '@utils/Storages'; +import NativeFile from '@modules/native-file'; +import type { RestoredNovelMapping } from '@database/types'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; + +export const getNovelFilesRestorePath = (cacheDirPath: string) => + `${cacheDirPath}/RestoredNovelFiles`; + +export const getLegacyFilesRestorePath = (cacheDirPath: string) => + `${cacheDirPath}/RestoredLegacyFiles`; + +const moveDirectoryContents = async (source: string, destination: string) => { + if (!(await NativeFile.exists(source))) { + return; + } + await NativeFile.mkdir(destination); + for (const item of await NativeFile.readDir(source)) { + const destinationPath = `${destination}/${item.name}`; + if (item.isDirectory) { + await moveDirectoryContents(item.path, destinationPath); + } else { + await NativeFile.moveFile(item.path, destinationPath); + } + } +}; + +const RESTORE_CHAPTER_LOOKUP_BATCH_SIZE = 100; + +export const restoreNovelFiles = async ( + stagingPath: string, + novelMappings: RestoredNovelMapping[], + restoreRunId: string, + onProgress?: (completed: number, total: number) => void, +) => { + const total = novelMappings.length + 1; + for (const [index, mapping] of novelMappings.entries()) { + const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; + if (!(await NativeFile.exists(sourceNovelPath))) { + onProgress?.(index + 1, total); + continue; + } + + const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; + await NativeFile.mkdir(destinationNovelPath); + const items = await NativeFile.readDir(sourceNovelPath); + for (const item of items) { + if (!item.isDirectory) { + await NativeFile.moveFile( + item.path, + `${destinationNovelPath}/${item.name}`, + ); + } + } + + const chapterItems = items.filter(item => item.isDirectory); + for ( + let start = 0; + start < chapterItems.length; + start += RESTORE_CHAPTER_LOOKUP_BATCH_SIZE + ) { + const chapterBatch = chapterItems.slice( + start, + start + RESTORE_CHAPTER_LOOKUP_BATCH_SIZE, + ); + const backupChapterIds = chapterBatch + .map(item => Number(item.name)) + .filter(id => Number.isInteger(id) && id > 0); + const chapterMappings = await getRestoreChapterMappings( + restoreRunId, + mapping.backupNovelId, + backupChapterIds, + ); + const restoredChapterIds = new Map( + chapterMappings.map(chapter => [ + String(chapter.backupChapterId), + chapter.restoredChapterId, + ]), + ); + for (const item of chapterBatch) { + const restoredChapterId = restoredChapterIds.get(item.name); + if (restoredChapterId !== undefined) { + await moveDirectoryContents( + item.path, + `${destinationNovelPath}/${restoredChapterId}`, + ); + } + } + } + onProgress?.(index + 1, total); + } + + if (await NativeFile.exists(stagingPath)) { + await NativeFile.unlink(stagingPath); + } + onProgress?.(total, total); +}; + +export const restoreLegacyFiles = async ( + stagingPath: string, + novelMappings: RestoredNovelMapping[], + restoreRunId: string, + onProgress?: (completed: number, total: number) => void, +) => { + const total = novelMappings.length + 2; + await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); + await restoreNovelFiles( + `${stagingPath}/Novels`, + novelMappings, + restoreRunId, + onProgress ? completed => onProgress(completed, total) : undefined, + ); + if (await NativeFile.exists(stagingPath)) { + await NativeFile.unlink(stagingPath); + } + onProgress?.(total, total); +}; diff --git a/src/services/backup/restore/index.ts b/src/services/backup/restore/index.ts new file mode 100644 index 0000000000..85a0c3f851 --- /dev/null +++ b/src/services/backup/restore/index.ts @@ -0,0 +1,318 @@ +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { _restoreCategory } from '@database/queries/CategoryQueries'; +import type { BackupCategory } from '@database/types'; +import type { RestoreProgressReporter } from './progress'; +import NativeFile from '@modules/native-file'; +import { getString } from '@i18n/translations'; +import { MMKVStorage } from '@utils/mmkv/mmkv'; +import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; +import type { PluginItem } from '@plugins/types'; +import { DEFAULT_BACKUP_OPTIONS, resolveBackupOptions } from '../options'; +import { + BackupEntryName, + type BackupManifest, + type ResolvedBackupManifest, +} from '../types'; +import { restoreNovels } from './novels'; +import type { RestoreResult } from './result'; + +const parsePluginList = (value: unknown): PluginItem[] => { + const parsed: unknown = typeof value === 'string' ? JSON.parse(value) : value; + if (!Array.isArray(parsed)) { + throw new Error('Invalid installed plugin registry'); + } + return parsed as PluginItem[]; +}; + +const restoreMMKVData = (data: Record) => { + for (const key in data) { + const value = data[key]; + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' + ) { + throw new Error('Invalid backup setting'); + } + MMKVStorage.set(key, value); + } +}; + +const getBackupManifest = async ( + cacheDirPath: string, +): Promise => { + try { + const fileContent = await NativeFile.readFile( + cacheDirPath + '/' + BackupEntryName.VERSION, + ); + const data = JSON.parse(fileContent) as Partial & { + version?: string; + }; + if ( + (data.formatVersion === 2 || data.formatVersion === 3) && + data.sections + ) { + const novelDataFormat = + data.novelDataFormat === 1 || data.novelDataFormat === 2 + ? data.novelDataFormat + : undefined; + const novelCount = + typeof data.novelCount === 'number' && + Number.isSafeInteger(data.novelCount) && + data.novelCount >= 0 + ? data.novelCount + : undefined; + return { + appVersion: data.appVersion ?? data.version ?? '', + formatVersion: data.formatVersion, + ...(novelDataFormat === undefined ? {} : { novelDataFormat }), + ...((data.formatVersion === 3 || novelDataFormat === 2) && + novelCount !== undefined + ? { novelCount } + : {}), + sections: resolveBackupOptions(data.sections), + }; + } + + return { + appVersion: data.version, + formatVersion: 1, + sections: DEFAULT_BACKUP_OPTIONS, + }; + } catch { + return { + formatVersion: 1, + sections: DEFAULT_BACKUP_OPTIONS, + }; + } +}; + +const createRestoreRunId = () => + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + +type RestoreBenchmarkLogger = (message: string) => void; + +const restoreDataInternal = async ( + cacheDirPath: string, + progressReporter: RestoreProgressReporter | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + restoreRunId: string, +): Promise => { + progressReporter?.( + 'manifest', + 0, + getString('backupScreen.restoringData'), + true, + ); + const manifest = await getBackupManifest(cacheDirPath); + benchmarkLog?.('restoreData:manifest:loaded'); + progressReporter?.( + 'manifest', + 1, + getString('backupScreen.restoringData'), + true, + ); + const novelSummary = manifest.sections.library + ? await restoreNovels( + cacheDirPath, + manifest, + restoreRunId, + progressReporter, + benchmarkLog, + ) + : { + novelCount: 0, + failedNovelCount: 0, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + novelIdMap: new Map(), + }; + if (!manifest.sections.library) { + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + } + let failedSectionCount = novelSummary.failedSectionCount; + const novelIdMap = novelSummary.novelIdMap; + const installedPluginsBeforeRestore = (() => { + try { + return parsePluginList( + MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', + ); + } catch { + return []; + } + })(); + let pluginsFromSettings: PluginItem[] = []; + + benchmarkLog?.('restoreData:categories:start'); + const categoriesProgressText = getString('backupScreen.restoringCategories'); + let categoryCount = 0; + let failedCategoryCount = 0; + if (!manifest.sections.library) { + progressReporter?.('categories', 1, categoriesProgressText, true); + } else { + progressReporter?.('categories', 0, categoriesProgressText, true); + const categoryFilePath = cacheDirPath + '/' + BackupEntryName.CATEGORY; + let categories: BackupCategory[] | undefined; + if (!(await NativeFile.exists(categoryFilePath))) { + failedSectionCount++; + } else { + try { + const fileContent = await NativeFile.readFile(categoryFilePath); + const parsed: unknown = JSON.parse(fileContent); + if (!Array.isArray(parsed)) { + throw new Error('Invalid backup categories'); + } + categories = parsed as BackupCategory[]; + } catch { + failedSectionCount++; + } + } + + if (categories) { + for (const [index, category] of categories.entries()) { + try { + await _restoreCategory( + { + ...category, + novelIds: category.novelIds.filter(novelId => + novelIdMap.has(novelId), + ), + }, + novelIdMap, + ); + categoryCount++; + } catch { + failedCategoryCount++; + } + progressReporter?.( + 'categories', + (index + 1) / Math.max(1, categories.length), + getString('backupScreen.restoringCategoriesProgress', { + current: index + 1, + total: categories.length, + }), + index + 1 === categories.length, + ); + } + } + progressReporter?.('categories', 1, categoriesProgressText, true); + } + benchmarkLog?.( + `restoreData:categories:done count=${categoryCount} failed=${failedCategoryCount}`, + ); + + benchmarkLog?.('restoreData:settings:start'); + const settingsProgressText = getString('backupScreen.restoringSettings'); + const settingsFilePath = cacheDirPath + '/' + BackupEntryName.SETTING; + let settingsRestored = !manifest.sections.settings; + + if (!manifest.sections.settings) { + progressReporter?.('settings', 1, settingsProgressText, true); + } else { + progressReporter?.('settings', 0, settingsProgressText, true); + if (!(await NativeFile.exists(settingsFilePath))) { + // Reported as a settings warning in the completion summary. + } else { + try { + const fileContent = await NativeFile.readFile(settingsFilePath); + const settingsData: Record = JSON.parse(fileContent); + if (INSTALLED_PLUGINS_KEY in settingsData) { + pluginsFromSettings = parsePluginList( + settingsData[INSTALLED_PLUGINS_KEY], + ); + delete settingsData[INSTALLED_PLUGINS_KEY]; + } + restoreMMKVData(settingsData); + settingsRestored = true; + } catch { + // Included in the completion warning below. + } + } + progressReporter?.('settings', 1, settingsProgressText, true); + } + benchmarkLog?.(`restoreData:settings:done restored=${settingsRestored}`); + + benchmarkLog?.('restoreData:plugins:start'); + let restoredPlugins = pluginsFromSettings; + const pluginsProgressText = getString('backupScreen.restoringPlugins'); + if (!manifest.sections.plugins) { + progressReporter?.('plugins', 1, pluginsProgressText, true); + } else { + progressReporter?.('plugins', 0, pluginsProgressText, true); + if (manifest.formatVersion === 2 || manifest.formatVersion === 3) { + const pluginMetadataPath = + cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; + if (!(await NativeFile.exists(pluginMetadataPath))) { + failedSectionCount++; + } else { + try { + restoredPlugins = parsePluginList( + await NativeFile.readFile(pluginMetadataPath), + ); + } catch { + failedSectionCount++; + } + } + } + const mergedPlugins = [ + ...new Map( + [...installedPluginsBeforeRestore, ...restoredPlugins].map(plugin => [ + plugin.id, + plugin, + ]), + ).values(), + ]; + MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); + progressReporter?.('plugins', 1, pluginsProgressText, true); + } + benchmarkLog?.(`restoreData:plugins:done count=${restoredPlugins.length}`); + benchmarkLog?.('restoreData:done'); + + return { + novelCount: novelSummary.novelCount, + failedNovelCount: novelSummary.failedNovelCount, + categoryCount, + failedCategoryCount, + settingsRestored, + failedSectionCount, + pluginIds: novelSummary.pluginIds, + novelMappings: novelSummary.novelMappings, + restoreRunId, + manifest, + }; +}; + +export const clearRestoreChapterMappingsSafely = async ( + restoreRunId: string, +): Promise => { + try { + await clearRestoreChapterMappings(restoreRunId); + } catch { + // Restore mappings are run-scoped and do not affect later restores. + } +}; + +export const restoreData = async ( + cacheDirPath: string, + progressReporter?: RestoreProgressReporter, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const restoreRunId = createRestoreRunId(); + try { + return await restoreDataInternal( + cacheDirPath, + progressReporter, + benchmarkLog, + restoreRunId, + ); + } catch (error) { + await clearRestoreChapterMappingsSafely(restoreRunId); + throw error; + } +}; diff --git a/src/services/backup/restore/local.ts b/src/services/backup/restore/local.ts new file mode 100644 index 0000000000..42d48a074c --- /dev/null +++ b/src/services/backup/restore/local.ts @@ -0,0 +1,106 @@ +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import NativeZipArchive from '@modules/native-zip-archive'; +import NativeFile from '@modules/native-file'; +import { ZipBackupName } from '../types'; +import { getString } from '@i18n/translations'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +const printTime = (time?: number) => { + if (!time) return ''; + if (time >= 1000) { + return (time / 1000).toFixed(2) + 's'; + } + return time.toFixed(2) + 'ms'; +}; + +let startTime: number; +let lastTime: number | undefined; +const logRestoreBenchmark = (message: string) => { + if (!__DEV__) { + return; + } + const currentTime = performance.now(); + if (message === 'local:start') { + lastTime = undefined; + startTime = currentTime; + } + const diff = lastTime ? printTime(currentTime - lastTime) : undefined; + lastTime = currentTime; + // Benchmark output is consumed from the Metro client log. + // eslint-disable-next-line no-console + console.log( + `[restore-benchmark] ${currentTime.toFixed(2)},${ + !diff ? '' : ' duration: ' + diff.padEnd(15, ' ') + } ${message}`, + ); + if (message === 'local:finalize:done') { + // eslint-disable-next-line no-console + console.log('Total time:', printTime(currentTime - startTime)); + lastTime = undefined; + } +}; + +export const restoreBackup = async ( + { sourceUri }: { sourceUri: string }, + setMeta?: TaskProgressUpdater, +) => { + logRestoreBenchmark('local:start'); + try { + await runRestore( + { + kind: 'local', + prepare: async report => { + report?.('source', 0, getString('backupScreen.copyingBackup'), true); + await clearBackupCache(); + const localPath = `${CACHE_DIR_PATH}-source.zip`; + await NativeFile.copyFile(sourceUri, localPath); + logRestoreBenchmark('local:copy:done'); + report?.('source', 1, getString('backupScreen.copyingBackup'), true); + report?.( + 'extract', + 0, + getString('backupScreen.extractingBackup'), + true, + ); + const outerArchiveStats = await NativeZipArchive.unzip( + localPath, + CACHE_DIR_PATH, + ); + logRestoreBenchmark( + `local:outer-unzip:done ${JSON.stringify(outerArchiveStats)}`, + ); + report?.( + 'extract', + 1, + getString('backupScreen.extractingBackup'), + true, + ); + }, + loadArchive: async (name, destination) => { + const archivePath = `${CACHE_DIR_PATH}/${name}`; + if (!(await NativeFile.exists(archivePath))) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + const archiveStats = await NativeZipArchive.unzip( + archivePath, + destination, + ); + logRestoreBenchmark( + `${ + name === ZipBackupName.DOWNLOAD + ? 'local:legacy-archive' + : 'local:selected-archive' + }-unzip:done ${JSON.stringify(archiveStats)}`, + ); + }, + benchmarkLog: logRestoreBenchmark, + }, + setMeta, + ); + } catch (error) { + lastTime = undefined; + setMeta?.(meta => ({ ...meta, isRunning: false })); + throw error; + } +}; diff --git a/src/services/backup/restore/novels.ts b/src/services/backup/restore/novels.ts new file mode 100644 index 0000000000..791b5edc6f --- /dev/null +++ b/src/services/backup/restore/novels.ts @@ -0,0 +1,515 @@ +import { + _restoreNovelAndChapters, + _restoreNovelsAndChapters, + recordRestoreFallback, + type RestoreNovelMetrics, + type RestoreNovelOptions, + type RestoreNovelProgress, +} from '@database/queries/NovelRestoreQueries'; +import type { BackupNovel, RestoredNovelMapping } from '@database/types'; +import NativeFile from '@modules/native-file'; +import { getString } from '@i18n/translations'; +import { NOVEL_STORAGE, ROOT_STORAGE } from '@utils/Storages'; +import type { RestoreProgressReporter } from './progress'; +import { BackupEntryName, type ResolvedBackupManifest } from '../types'; +import { + NovelFileValidationError, + decodeAndValidateNovelFile, +} from './validation'; + +const RESTORE_NOVEL_BATCH_SIZE = 100; +const BACKUP_FILE_CONCURRENCY = 8; +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +type BackupNovelFileDescriptor = { + name: string; + path: string; +}; + +type RestoreBenchmarkLogger = (message: string) => void; + +type RestoreNovelTelemetry = { + database: RestoreNovelMetrics; + collectDatabaseMetrics: boolean; + uniqueInputChapterCount: number; +}; + +export type NovelRestoreSummary = { + novelCount: number; + failedNovelCount: number; + failedSectionCount: number; + pluginIds: string[]; + novelMappings: RestoredNovelMapping[]; + novelIdMap: Map; +}; + +const restoreNovelsWithTelemetry = async ( + cacheDirPath: string, + manifest: ResolvedBackupManifest, + restoreRunId: string, + progressReporter: RestoreProgressReporter | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + telemetry: RestoreNovelTelemetry, +): Promise => { + const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; + const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; + const summary: NovelRestoreSummary = { + novelCount: 0, + failedNovelCount: 0, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + novelIdMap: new Map(), + }; + + benchmarkLog?.('restoreData:novels:validation:start'); + progressReporter?.( + 'novels', + 0, + getString('backupScreen.validatingNovels'), + true, + ); + if (!(await NativeFile.exists(novelDirPath))) { + summary.failedSectionCount++; + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + return summary; + } + + let items: BackupNovelFileDescriptor[]; + try { + items = (await NativeFile.readDir(novelDirPath)) + .filter(item => !item.isDirectory) + .sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ) + .map(item => ({ name: item.name, path: item.path })); + } catch { + summary.failedSectionCount++; + progressReporter?.( + 'novels', + 1, + getString('backupScreen.restoringNovels'), + true, + ); + return summary; + } + + const seenNovelIds = new Set(); + const seenNovelIdentities = new Set(); + const seenChapterIdentities = new Set(); + const pluginIds = new Set(); + const pendingNovels: BackupNovel[] = []; + let filesProcessed = 0; + let inputRecordCount = 0; + let persistedNovelCount = 0; + const totalNovelCount = + manifest.formatVersion === 3 || + (manifest.formatVersion === 2 && manifest.novelDataFormat === 2) + ? manifest.novelCount + : items.length; + let completedFileWork = 0; + let pendingFileWork = 0; + let activeFileWork = 0; + let activeWorkFraction = 0; + let activeDatabaseStage: RestoreNovelProgress['stage'] | undefined; + let progressInterruptionLatched = false; + let progressInterruption: unknown; + let readMs = 0; + let parseMs = 0; + let databaseMs = 0; + let coverProcessingMs = 0; + let coverCandidates = 0; + let coverFilesFound = 0; + let coverFilesMissing = 0; + let coverFilesCopied = 0; + let coverProcessingFailures = 0; + + const getNovelCountText = () => + getString('backupScreen.restoringNovelsProgress', { + current: persistedNovelCount, + total: totalNovelCount ?? inputRecordCount, + }); + const getNovelPhaseFraction = () => { + if (items.length === 0) { + return 1; + } + return ( + (0.25 * filesProcessed + + 0.75 * (completedFileWork + activeFileWork * activeWorkFraction)) / + items.length + ); + }; + const publishNovelProgress = (force = false) => { + if (progressInterruptionLatched) { + return; + } + try { + progressReporter?.( + 'novels', + getNovelPhaseFraction(), + getNovelCountText(), + force, + ); + } catch (error) { + if (!progressInterruptionLatched) { + progressInterruptionLatched = true; + progressInterruption = error; + } + throw error; + } + }; + const processFile = async (item: BackupNovelFileDescriptor) => { + const readStartedAt = performance.now(); + let fileContent: string | undefined; + try { + fileContent = await NativeFile.readFile(item.path); + } catch { + summary.failedNovelCount++; + } finally { + readMs += performance.now() - readStartedAt; + } + + if (fileContent === undefined) { + inputRecordCount++; + completedFileWork++; + } else { + const parseStartedAt = performance.now(); + try { + const novels = decodeAndValidateNovelFile( + fileContent, + manifest, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + inputRecordCount += novels.length; + if (novels.length === 0) { + completedFileWork++; + } else { + pendingFileWork++; + } + for (const novel of novels) { + pluginIds.add(novel.pluginId); + pendingNovels.push(novel); + } + if (telemetry.collectDatabaseMetrics) { + telemetry.uniqueInputChapterCount = seenChapterIdentities.size; + } + } catch (error) { + const rejectedRecordCount = + error instanceof NovelFileValidationError ? error.recordCount : 1; + summary.failedNovelCount += rejectedRecordCount; + inputRecordCount += rejectedRecordCount; + completedFileWork++; + } finally { + parseMs += performance.now() - parseStartedAt; + } + } + + filesProcessed++; + publishNovelProgress(); + }; + + publishNovelProgress(true); + + const restoreNovelBatch = async () => { + if (pendingNovels.length === 0) { + return; + } + const batch = pendingNovels.splice(0, pendingNovels.length); + const fileWorkForBatch = pendingFileWork; + pendingFileWork = 0; + activeFileWork = fileWorkForBatch; + activeWorkFraction = 0; + activeDatabaseStage = undefined; + + const reportDatabaseCheckpoint = ( + progress: RestoreNovelProgress, + fallbackIndex?: number, + ) => { + if (progressInterruptionLatched) { + return; + } + try { + const stageFraction = + progress.total > 0 + ? Math.max(0, Math.min(1, progress.completed / progress.total)) + : 1; + let databaseFraction: number; + if (progress.stage === 'novels') { + databaseFraction = 0.1 * stageFraction; + } else if (progress.stage === 'chapters') { + databaseFraction = 0.1 + 0.8 * stageFraction; + } else { + databaseFraction = 0.9 + 0.1 * stageFraction; + } + if (fallbackIndex !== undefined) { + databaseFraction = (fallbackIndex + databaseFraction) / batch.length; + } + activeWorkFraction = Math.max( + activeWorkFraction, + 0.8 * databaseFraction, + ); + const stageChanged = activeDatabaseStage !== progress.stage; + activeDatabaseStage = progress.stage; + const stageCompleted = + progress.total <= 0 || progress.completed >= progress.total; + publishNovelProgress(stageChanged || stageCompleted); + } catch (error) { + if (!progressInterruptionLatched) { + progressInterruptionLatched = true; + progressInterruption = error; + } + } + }; + const onBatchProgress = (progress: RestoreNovelProgress) => + reportDatabaseCheckpoint(progress); + const restoreOptions: RestoreNovelOptions = { + includeChapterMappings: manifest.sections.downloadedFiles, + ...(manifest.sections.downloadedFiles ? { restoreRunId } : {}), + ...(progressReporter ? { onProgress: onBatchProgress } : {}), + }; + let restoredNovels: { + backupNovel: BackupNovel; + mapping: RestoredNovelMapping; + }[] = []; + const databaseStartedAt = performance.now(); + try { + try { + const mappings = telemetry.collectDatabaseMetrics + ? await _restoreNovelsAndChapters( + batch, + restoreOptions, + telemetry.database, + ) + : await _restoreNovelsAndChapters(batch, restoreOptions); + if (mappings.length !== batch.length) { + throw new Error('Restore returned incomplete novel mappings'); + } + restoredNovels = batch.map((backupNovel, index) => ({ + backupNovel, + mapping: mappings[index], + })); + persistedNovelCount += batch.length; + } catch (databaseError) { + if (progressInterruptionLatched) { + throw progressInterruption; + } + if (telemetry.collectDatabaseMetrics) { + telemetry.database.novelBatchFallbacks++; + recordRestoreFallback(telemetry.database, databaseError); + } + for (const [index, backupNovel] of batch.entries()) { + if (telemetry.collectDatabaseMetrics) { + telemetry.database.novelFallbackRowsAttempted++; + } + const fallbackOptions: RestoreNovelOptions = progressReporter + ? { + ...restoreOptions, + onProgress: progress => + reportDatabaseCheckpoint(progress, index), + } + : restoreOptions; + let mapping: RestoredNovelMapping; + try { + mapping = telemetry.collectDatabaseMetrics + ? await _restoreNovelAndChapters( + backupNovel, + fallbackOptions, + telemetry.database, + ) + : await _restoreNovelAndChapters(backupNovel, fallbackOptions); + } catch (fallbackError) { + if (progressInterruptionLatched) { + throw progressInterruption; + } + if (telemetry.collectDatabaseMetrics) { + recordRestoreFallback(telemetry.database, fallbackError); + } + summary.failedNovelCount++; + continue; + } + restoredNovels.push({ backupNovel, mapping }); + persistedNovelCount++; + if (progressInterruptionLatched) { + throw progressInterruption; + } + publishNovelProgress(true); + } + } + } finally { + databaseMs += performance.now() - databaseStartedAt; + } + + if (progressInterruptionLatched) { + throw progressInterruption; + } + activeWorkFraction = 0.8; + activeDatabaseStage = undefined; + publishNovelProgress(true); + + const shouldRestoreCover = (backupNovel: BackupNovel) => + !manifest.sections.downloadedFiles && + backupNovel.cover?.startsWith(APP_STORAGE_URI); + const hasCoverWork = restoredNovels.some(({ backupNovel }) => + shouldRestoreCover(backupNovel), + ); + const coverStartedAt = performance.now(); + if (hasCoverWork) { + let attemptedCoverRecords = 0; + for ( + let start = 0; + start < restoredNovels.length; + start += BACKUP_FILE_CONCURRENCY + ) { + const coverBatch = restoredNovels.slice( + start, + start + BACKUP_FILE_CONCURRENCY, + ); + await Promise.all( + coverBatch.map(async ({ backupNovel, mapping: novelMapping }) => { + try { + if (shouldRestoreCover(backupNovel)) { + coverCandidates++; + const coverBackupPath = coversDirPath + '/' + backupNovel.id; + if (await NativeFile.exists(coverBackupPath)) { + coverFilesFound++; + const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; + await NativeFile.mkdir( + coverPath.slice(0, Math.max(0, coverPath.lastIndexOf('/'))), + ); + await NativeFile.copyFile(coverBackupPath, coverPath); + coverFilesCopied++; + } else { + coverFilesMissing++; + } + } + } catch { + coverProcessingFailures++; + summary.failedNovelCount++; + } + }), + ); + attemptedCoverRecords += coverBatch.length; + activeWorkFraction = + 0.8 + (0.2 * attemptedCoverRecords) / restoredNovels.length; + publishNovelProgress(attemptedCoverRecords === restoredNovels.length); + } + } + coverProcessingMs += performance.now() - coverStartedAt; + completedFileWork += activeFileWork; + activeFileWork = 0; + activeWorkFraction = 0; + publishNovelProgress(true); + + for (const { backupNovel, mapping: novelMapping } of restoredNovels) { + summary.novelMappings.push(novelMapping); + summary.novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); + summary.novelCount++; + } + }; + + let nextFileIndex = 0; + while (nextFileIndex < items.length || pendingNovels.length > 0) { + if (pendingNovels.length >= RESTORE_NOVEL_BATCH_SIZE) { + const writePromise = restoreNovelBatch(); + if (nextFileIndex < items.length) { + const filePromise = processFile(items[nextFileIndex++]); + const [writeResult, fileResult] = await Promise.allSettled([ + writePromise, + filePromise, + ]); + if (fileResult.status === 'rejected') { + throw fileResult.reason; + } + if (writeResult.status === 'rejected') { + throw writeResult.reason; + } + } else { + await writePromise; + } + continue; + } + + if (nextFileIndex < items.length) { + await processFile(items[nextFileIndex++]); + continue; + } + + await restoreNovelBatch(); + } + publishNovelProgress(true); + + benchmarkLog?.(`restoreData:novels:pipeline:done total=${filesProcessed}`); + benchmarkLog?.( + `restoreData:novels:done count=${summary.novelCount} failed=${ + summary.failedNovelCount + } readMs=${readMs.toFixed(1)} parseMs=${parseMs.toFixed( + 1, + )} databaseMs=${databaseMs.toFixed( + 1, + )} coverProcessingMs=${coverProcessingMs.toFixed( + 1, + )} coverCandidates=${coverCandidates} coverFilesFound=${coverFilesFound} coverFilesMissing=${coverFilesMissing} coverFilesCopied=${coverFilesCopied} coverProcessingFailures=${coverProcessingFailures}`, + ); + + summary.pluginIds = [...pluginIds]; + return summary; +}; + +export const restoreNovels = async ( + cacheDirPath: string, + manifest: ResolvedBackupManifest, + restoreRunId: string, + progressReporter?: RestoreProgressReporter, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const telemetry: RestoreNovelTelemetry = { + collectDatabaseMetrics: __DEV__ && benchmarkLog !== undefined, + database: { + novelUpsertCalls: 0, + novelUpsertRowsAttempted: 0, + novelUpsertMs: 0, + novelIdentityLookupCalls: 0, + novelIdentityLookupRowsReturned: 0, + novelIdentityLookupMs: 0, + chapterWriteChunkCalls: 0, + chapterWriteRowsAttempted: 0, + chapterWriteMs: 0, + chapterMappingRowsAttempted: 0, + statsRefreshCalls: 0, + statsRefreshNovels: 0, + statsRefreshMs: 0, + novelBatchFallbacks: 0, + novelFallbackRowsAttempted: 0, + chapterBatchFallbacks: 0, + chapterFallbackRowsAttempted: 0, + fallbackCauses: [], + }, + uniqueInputChapterCount: 0, + }; + try { + return await restoreNovelsWithTelemetry( + cacheDirPath, + manifest, + restoreRunId, + progressReporter, + benchmarkLog, + telemetry, + ); + } finally { + if (telemetry.collectDatabaseMetrics) { + benchmarkLog?.( + `restoreData:database:summary ${JSON.stringify({ + uniqueInputChapterCount: telemetry.uniqueInputChapterCount, + ...telemetry.database, + })}`, + ); + } + } +}; diff --git a/src/services/backup/restore/progress.ts b/src/services/backup/restore/progress.ts new file mode 100644 index 0000000000..9ab0bd8b24 --- /dev/null +++ b/src/services/backup/restore/progress.ts @@ -0,0 +1,109 @@ +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; + +export type RestorePhase = + | 'source' + | 'extract' + | 'manifest' + | 'novels' + | 'categories' + | 'settings' + | 'plugins' + | 'selectedFiles' + | 'finalize'; + +export type RestoreProgressReporter = ( + phase: RestorePhase, + fraction: number, + progressText: string, + force?: boolean, +) => void; + +const PROGRESS_UPDATE_INTERVAL_MS = 250; + +const LOCAL_PHASE_WEIGHTS: Record = { + source: 0.09, + extract: 0.21, + manifest: 0.002, + novels: 0.68, + categories: 0.004, + settings: 0.002, + plugins: 0.004, + selectedFiles: 0.005, + finalize: 0.003, +}; + +const PHASES = Object.keys(LOCAL_PHASE_WEIGHTS) as RestorePhase[]; + +export const createRestoreProgressReporter = ( + setMeta: TaskProgressUpdater | undefined, + source: 'local' | 'remote', +): RestoreProgressReporter | undefined => { + if (!setMeta) { + return undefined; + } + + const weights = { ...LOCAL_PHASE_WEIGHTS }; + if (source === 'remote') { + weights.source = 0.3; + weights.extract = 0; + } + + const phaseStarts = {} as Record; + let nextStart = 0; + for (const phase of PHASES) { + phaseStarts[phase] = nextStart; + nextStart += weights[phase]; + } + + let highWaterMark = 0; + let currentPhase: RestorePhase | undefined; + let lastEmittedProgress: number | undefined; + let lastEmittedText: string | undefined; + let lastEmittedAt = Number.NEGATIVE_INFINITY; + + return (phase, fraction, progressText, force = false) => { + const normalizedFraction = Number.isNaN(fraction) + ? 0 + : Math.max(0, Math.min(1, fraction)); + const mappedProgress = Math.max( + 0, + Math.min( + phase === 'finalize' && normalizedFraction === 1 + ? 1 + : 1 - Number.EPSILON, + phaseStarts[phase] + weights[phase] * normalizedFraction, + ), + ); + highWaterMark = Math.max(highWaterMark, mappedProgress); + + const phaseChanged = currentPhase !== phase; + currentPhase = phase; + + if ( + lastEmittedProgress === highWaterMark && + lastEmittedText === progressText + ) { + return; + } + + const now = Date.now(); + const completed = normalizedFraction === 1; + if ( + !phaseChanged && + !force && + !completed && + now - lastEmittedAt < PROGRESS_UPDATE_INTERVAL_MS + ) { + return; + } + + lastEmittedProgress = highWaterMark; + lastEmittedText = progressText; + lastEmittedAt = now; + setMeta(meta => ({ + ...meta, + progress: highWaterMark, + progressText, + })); + }; +}; diff --git a/src/services/backup/restoreResult.ts b/src/services/backup/restore/result.ts similarity index 97% rename from src/services/backup/restoreResult.ts rename to src/services/backup/restore/result.ts index dc11bd9880..24327ff913 100644 --- a/src/services/backup/restoreResult.ts +++ b/src/services/backup/restore/result.ts @@ -5,8 +5,8 @@ import { reloadInstalledPlugins, } from '@plugins/pluginManager'; import { PLUGIN_STORAGE } from '@utils/Storages'; -import type { ResolvedBackupManifest } from './types'; import type { RestoredNovelMapping } from '@database/types'; +import type { ResolvedBackupManifest } from '../types'; export type RestoreResult = { novelCount: number; @@ -17,6 +17,7 @@ export type RestoreResult = { failedSectionCount: number; pluginIds: string[]; novelMappings: RestoredNovelMapping[]; + restoreRunId: string; manifest: ResolvedBackupManifest; }; diff --git a/src/services/backup/restore/run.ts b/src/services/backup/restore/run.ts new file mode 100644 index 0000000000..b199880fdc --- /dev/null +++ b/src/services/backup/restore/run.ts @@ -0,0 +1,146 @@ +import { getString } from '@i18n/translations'; +import NativeFile from '@modules/native-file'; +import { BackupEntryName, ZipBackupName } from '../types'; +import { getSelectedBackupFileSections } from '../fileSections'; +import { CACHE_DIR_PATH } from '../cache'; +import { + getLegacyFilesRestorePath, + getNovelFilesRestorePath, + restoreLegacyFiles, + restoreNovelFiles, +} from './files'; +import { restoreData, clearRestoreChapterMappingsSafely } from './index'; +import { + finalizeRestoredPlugins, + getRestoreCompletionText, + type RestoreResult, +} from './result'; +import { + createRestoreProgressReporter, + type RestoreProgressReporter, +} from './progress'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; + +type RestoreSource = { + kind: 'local' | 'remote'; + prepare: (report: RestoreProgressReporter | undefined) => Promise; + loadArchive: (name: ZipBackupName, destination: string) => Promise; + benchmarkLog?: (message: string) => void; +}; + +export const runRestore = async ( + source: RestoreSource, + setMeta?: TaskProgressUpdater, +): Promise => { + const progressReporter = createRestoreProgressReporter(setMeta, source.kind); + let restoreResult: RestoreResult | undefined; + + setMeta?.(meta => ({ ...meta, isRunning: true })); + + try { + await source.prepare(progressReporter); + restoreResult = await restoreData( + CACHE_DIR_PATH, + progressReporter, + source.benchmarkLog, + ); + source.benchmarkLog?.('local:restore-data:done'); + + const selectedFilesText = getString('backupScreen.restoringSelectedFiles'); + const publishSelectedFilesProgress = (fraction: number, force = false) => { + progressReporter?.('selectedFiles', fraction, selectedFilesText, force); + }; + const reportSelectedFileMoves = (completed: number, total: number) => { + publishSelectedFilesProgress( + 0.5 + 0.5 * (total > 0 ? completed / total : 1), + completed >= total, + ); + }; + publishSelectedFilesProgress(0, true); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); + await source.loadArchive(ZipBackupName.DOWNLOAD, legacyFilesRestorePath); + publishSelectedFilesProgress(0.5, true); + await restoreLegacyFiles( + legacyFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + reportSelectedFileMoves, + ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + const sections = getSelectedBackupFileSections( + restoreResult.manifest.sections, + source.kind === 'local' ? restoreResult.manifest.formatVersion : 2, + ); + if (sections.length === 0) { + publishSelectedFilesProgress(0.5, true); + } + + let novelFilesSource: string | undefined; + if ( + source.kind === 'local' && + restoreResult.manifest.formatVersion === 3 && + restoreResult.manifest.sections.downloadedFiles + ) { + novelFilesSource = `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}`; + if (!(await NativeFile.exists(novelFilesSource))) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + } + + for (const [index, section] of sections.entries()) { + const destination = + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath; + await source.loadArchive(section.archiveName, destination); + publishSelectedFilesProgress( + 0.5 * ((index + 1) / sections.length), + index + 1 === sections.length, + ); + } + source.benchmarkLog?.('local:selected-archives:done'); + + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesSource ?? novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + reportSelectedFileMoves, + ); + } else { + publishSelectedFilesProgress(1, true); + } + } + + source.benchmarkLog?.('local:downloaded-files:done'); + source.benchmarkLog?.('local:selected-files:done'); + + progressReporter?.( + 'finalize', + 0, + getString('backupScreen.finalizingRestore'), + true, + ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); + progressReporter?.('finalize', 1, completionText, true); + source.benchmarkLog?.('local:finalize:done'); + + setMeta?.(meta => ({ + ...meta, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); + } + } +}; diff --git a/src/services/backup/restore/selfhost.ts b/src/services/backup/restore/selfhost.ts new file mode 100644 index 0000000000..8dd98657e0 --- /dev/null +++ b/src/services/backup/restore/selfhost.ts @@ -0,0 +1,32 @@ +import { download } from '@api/remote'; +import { getString } from '@i18n/translations'; +import { CACHE_DIR_PATH, clearBackupCache } from '../cache'; +import { ZipBackupName } from '../types'; +import type { + SelfHostData, + TaskProgressUpdater, +} from '@services/backgroundTasks/contracts'; +import { runRestore } from './run'; + +export const selfHostRestore = async ( + { host, backupFolder }: SelfHostData, + setMeta: TaskProgressUpdater, +) => { + await runRestore( + { + kind: 'remote', + prepare: async report => { + const progressText = getString('backupScreen.downloadingData'); + report?.('source', 0, progressText, true); + await clearBackupCache(); + await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); + report?.('source', 1, progressText, true); + report?.('extract', 1, progressText, true); + }, + loadArchive: async (name, destination) => { + await download(host, backupFolder, name, destination); + }, + }, + setMeta, + ); +}; diff --git a/src/services/backup/restore/validation.ts b/src/services/backup/restore/validation.ts new file mode 100644 index 0000000000..ba35e3f2ad --- /dev/null +++ b/src/services/backup/restore/validation.ts @@ -0,0 +1,127 @@ +import type { BackupNovel } from '@database/types'; +import { ROOT_STORAGE } from '@utils/Storages'; +import { decodeNovelBatch, normalizeLegacyNovel } from '../novelPayload'; +import type { ResolvedBackupManifest } from '../types'; + +const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; + +export class NovelFileValidationError extends Error { + recordCount: number; + + constructor(recordCount: number, cause: unknown) { + super(cause instanceof Error ? cause.message : 'Invalid novel file'); + this.name = 'NovelFileValidationError'; + this.recordCount = recordCount; + } +} + +export const decodeRestoreNovelFile = ( + payload: unknown, + manifest: ResolvedBackupManifest, +) => { + const novels = + manifest.novelDataFormat === 2 + ? decodeNovelBatch(payload) + : [normalizeLegacyNovel(payload)]; + return novels.map(novel => { + const { + chaptersDownloaded: _chaptersDownloaded, + chaptersUnread: _chaptersUnread, + totalChapters: _totalChapters, + lastReadAt: _lastReadAt, + lastUpdatedAt: _lastUpdatedAt, + ...normalized + } = novel as BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; + }; + return normalized.cover && !normalized.cover.startsWith('http') + ? { ...normalized, cover: APP_STORAGE_URI + normalized.cover } + : normalized; + }); +}; + +export const getNovelFileRecordCount = ( + payload: unknown, + manifest: ResolvedBackupManifest, +) => { + if (manifest.novelDataFormat !== 2) { + return 1; + } + return Array.isArray(payload) ? Math.max(1, payload.length) : 1; +}; + +export const validateNovelFileRecords = ( + novels: BackupNovel[], + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + const fileNovelIds = new Set(); + const fileNovelIdentities = new Set(); + const fileChapterIdentities = new Set(); + for (const novel of novels) { + const novelIdentity = `${novel.pluginId}\u0000${novel.path}`; + if ( + fileNovelIds.has(novel.id) || + seenNovelIds.has(novel.id) || + fileNovelIdentities.has(novelIdentity) || + seenNovelIdentities.has(novelIdentity) + ) { + throw new Error('Duplicate backup novel identity'); + } + fileNovelIds.add(novel.id); + fileNovelIdentities.add(novelIdentity); + for (const chapter of novel.chapters) { + const chapterIdentity = `${novel.id}\u0000${chapter.path}`; + if ( + fileChapterIdentities.has(chapterIdentity) || + seenChapterIdentities.has(chapterIdentity) + ) { + throw new Error('Duplicate backup chapter identity'); + } + fileChapterIdentities.add(chapterIdentity); + } + } + for (const novelId of fileNovelIds) { + seenNovelIds.add(novelId); + } + for (const novelIdentity of fileNovelIdentities) { + seenNovelIdentities.add(novelIdentity); + } + for (const chapterIdentity of fileChapterIdentities) { + seenChapterIdentities.add(chapterIdentity); + } +}; + +export const decodeAndValidateNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + let payload: unknown; + try { + payload = JSON.parse(fileContent); + } catch (error) { + throw new NovelFileValidationError(1, error); + } + + const recordCount = getNovelFileRecordCount(payload, manifest); + try { + const novels = decodeRestoreNovelFile(payload, manifest); + validateNovelFileRecords( + novels, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + return novels; + } catch (error) { + throw new NovelFileValidationError(recordCount, error); + } +}; diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index b25a8194ec..8e7662db52 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -1,29 +1,15 @@ import { sleep } from '@utils/sleep'; -import { download, upload } from '@api/remote'; +import { upload } from '@api/remote'; import { getString } from '@i18n/translations'; -import { - CACHE_DIR_PATH, - clearBackupCache, - prepareBackupData, - restoreData, -} from '../utils'; -import { - finalizeRestoredPlugins, - getRestoreCompletionText, -} from '../restoreResult'; +import { CACHE_DIR_PATH } from '../cache'; +import { prepareBackupData } from '../create'; import { getBackupCompletionText } from '../backupResult'; import { ZipBackupName } from '../types'; import type { SelfHostData, TaskProgressUpdater, } from '@services/backgroundTasks/contracts'; -import { - getLegacyFilesRestorePath, - getNovelFilesRestorePath, - getSelectedBackupFileSections, - restoreLegacyFiles, - restoreNovelFiles, -} from '../fileSections'; +import { getSelectedBackupFileSections } from '../fileSections'; import { resolveBackupOptions } from '../options'; export const createSelfHostBackup = async ( @@ -58,7 +44,7 @@ export const createSelfHostBackup = async ( await sleep(200); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await upload(host, backupFolder, section.archiveName, section.storagePath); } @@ -71,83 +57,3 @@ export const createSelfHostBackup = async ( completionText, })); }; - -export const selfHostRestore = async ( - { host, backupFolder }: SelfHostData, - setMeta: TaskProgressUpdater, -) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download( - host, - backupFolder, - ZipBackupName.DOWNLOAD, - legacyFilesRestorePath, - ); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - await download( - host, - backupFolder, - section.archiveName, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, - restoreResult.novelMappings, - ); - } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); - - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); -}; diff --git a/src/services/backup/types.ts b/src/services/backup/types.ts index aefd5cf65f..35ce7a649d 100644 --- a/src/services/backup/types.ts +++ b/src/services/backup/types.ts @@ -14,11 +14,14 @@ export enum BackupEntryName { PLUGIN_METADATA = 'Plugins.json', COVERS = 'Covers', NOVEL_AND_CHAPTERS = 'NovelAndChapters', + NOVEL_FILES = 'NovelFiles', } export type BackupManifest = { appVersion: string; - formatVersion: 2; + formatVersion: 2 | 3; + novelDataFormat?: 1 | 2; + novelCount?: number; sections: BackupOptions; }; @@ -27,5 +30,6 @@ export type ResolvedBackupManifest = | { appVersion?: string; formatVersion: 1; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts deleted file mode 100644 index d7b186ef80..0000000000 --- a/src/services/backup/utils.ts +++ /dev/null @@ -1,453 +0,0 @@ -import { SELF_HOST_BACKUP } from '@hooks/persisted/useSelfHost'; -import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMigration'; -import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; -import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { version } from '../../../package.json'; -import { - _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; -import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; -import { - _restoreCategory, - getAllNovelCategories, - getCategoriesFromDb, -} from '@database/queries/CategoryQueries'; -import { - BackupCategory, - BackupNovel, - type RestoredNovelMapping, -} from '@database/types'; -import { - BackupEntryName, - type BackupManifest, - type ResolvedBackupManifest, -} from './types'; -import { NOVEL_STORAGE, ROOT_STORAGE } from '@utils/Storages'; -import { BACKGROUND_TASKS_STORE_KEY } from '@services/backgroundTasks/constants'; -import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; -import NativeFile from '@modules/native-file'; -import { getString } from '@i18n/translations'; -import type { RestoreResult } from './restoreResult'; -import type { BackupResult } from './backupResult'; -import { - DEFAULT_BACKUP_OPTIONS, - resolveBackupOptions, - type BackupOptions, -} from './options'; -import { INSTALLED_PLUGINS_KEY } from '@plugins/pluginManager'; -import type { PluginItem } from '@plugins/types'; - -const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; - -const stripUriSuffix = (uri: string) => uri.split(/[?#]/, 1)[0]; - -const parentDirectory = (path: string) => - path.slice(0, Math.max(0, path.lastIndexOf('/'))); - -const parsePluginList = (value: unknown): PluginItem[] => { - const parsed = typeof value === 'string' ? JSON.parse(value) : value; - if (!Array.isArray(parsed)) { - throw new Error('Invalid installed plugin registry'); - } - return parsed as PluginItem[]; -}; - -export const CACHE_DIR_PATH = - NativeFile.ExternalCachesDirectoryPath + '/BackupData'; - -export const clearBackupCache = async (cacheDirPath = CACHE_DIR_PATH) => { - if (await NativeFile.exists(cacheDirPath)) { - await NativeFile.unlink(cacheDirPath); - } -}; - -const backupMMKVData = () => { - const excludeKeys = [ - BACKGROUND_TASKS_STORE_KEY, - OLD_TRACKED_NOVEL_PREFIX, - SELF_HOST_BACKUP, - LAST_UPDATE_TIME, - INSTALLED_PLUGINS_KEY, - ]; - const keys = MMKVStorage.getAllKeys().filter( - key => !excludeKeys.includes(key), - ); - const data = {} as any; - for (const key of keys) { - let value: number | string | boolean | undefined = - MMKVStorage.getString(key); - if (!value) { - value = MMKVStorage.getBoolean(key); - } - if (key && value) { - data[key] = value; - } - } - return data; -}; - -const restoreMMKVData = (data: any) => { - for (const key in data) { - MMKVStorage.set(key, data[key]); - } -}; - -export const prepareBackupData = async ( - cacheDirPath: string, - requestedOptions?: BackupOptions, -): Promise => { - const options = resolveBackupOptions(requestedOptions); - const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; - const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; - let failedNovelCount = 0; - let failedSectionCount = 0; - - await clearBackupCache(cacheDirPath); - await NativeFile.mkdir(cacheDirPath); - - // version - const manifest: BackupManifest = { - appVersion: version, - formatVersion: 2, - sections: options, - }; - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.VERSION, - JSON.stringify(manifest), - ); - - // novels - if (options.library) { - await NativeFile.mkdir(novelDirPath); - await NativeFile.mkdir(coversDirPath); - await getAllNovels().then(async novels => { - for (const novel of novels) { - try { - const chapters = await getAllNovelChaptersForBackup(novel.id); - const backedUpChapters = options.downloadedFiles - ? chapters - : chapters.map(chapter => ({ - ...chapter, - isDownloaded: false, - })); - let cover = novel.cover; - if (cover?.startsWith(APP_STORAGE_URI)) { - try { - await NativeFile.copyFile( - stripUriSuffix(cover), - coversDirPath + '/' + novel.id, - ); - cover = cover.replace(APP_STORAGE_URI, ''); - } catch { - cover = options.downloadedFiles - ? cover.replace(APP_STORAGE_URI, '') - : null; - } - } - await NativeFile.writeFile( - novelDirPath + '/' + novel.id + '.json', - JSON.stringify({ - chapters: backedUpChapters, - ...novel, - cover, - }), - ); - } catch { - failedNovelCount++; - } - } - }); - - // categories - try { - const categories = await getCategoriesFromDb(); - const novelCategories = await getAllNovelCategories(); - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.CATEGORY, - JSON.stringify( - categories.map(category => { - return { - ...category, - novelIds: novelCategories - .filter(nc => nc.categoryId === category.id) - .map(nc => nc.novelId), - }; - }), - ), - ); - } catch { - failedSectionCount++; - } - } - - // settings - if (options.settings) { - try { - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.SETTING, - JSON.stringify(backupMMKVData()), - ); - } catch { - failedSectionCount++; - } - } - - // installed plugin registry - if (options.plugins) { - try { - await NativeFile.writeFile( - cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA, - MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', - ); - } catch { - failedSectionCount++; - } - } - - return { - failedNovelCount, - failedSectionCount, - }; -}; - -const getBackupManifest = async ( - cacheDirPath: string, -): Promise => { - try { - const fileContent = await NativeFile.readFile( - cacheDirPath + '/' + BackupEntryName.VERSION, - ); - const data = JSON.parse(fileContent) as Partial & { - version?: string; - }; - if (data.formatVersion === 2 && data.sections) { - return { - appVersion: data.appVersion ?? data.version ?? '', - formatVersion: 2, - sections: resolveBackupOptions(data.sections), - }; - } - - return { - appVersion: data.version, - formatVersion: 1, - sections: DEFAULT_BACKUP_OPTIONS, - }; - } catch { - return { - formatVersion: 1, - sections: DEFAULT_BACKUP_OPTIONS, - }; - } -}; - -const updateRestoreProgress = ( - setMeta: TaskProgressUpdater | undefined, - progressText: string, -) => { - setMeta?.(meta => ({ - ...meta, - progressText, - })); -}; - -export const restoreData = async ( - cacheDirPath: string, - setMeta?: TaskProgressUpdater, -): Promise => { - const manifest = await getBackupManifest(cacheDirPath); - const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; - const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; - const pluginIds = new Set(); - const novelIdMap = new Map(); - const novelMappings: RestoredNovelMapping[] = []; - const installedPluginsBeforeRestore = (() => { - try { - return parsePluginList( - MMKVStorage.getString(INSTALLED_PLUGINS_KEY) ?? '[]', - ); - } catch { - return []; - } - })(); - let pluginsFromSettings: PluginItem[] = []; - - // version - // nothing to do - - // novels - if (manifest.sections.library) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); - } - let novelCount = 0; - let failedCount = 0; - let failedSectionCount = 0; - - if (!manifest.sections.library) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(novelDirPath))) { - failedSectionCount++; - } else { - try { - const items = (await NativeFile.readDir(novelDirPath)).filter( - item => !item.isDirectory, - ); - for (const [index, item] of items.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringNovelsProgress', { - current: index + 1, - total: items.length, - }), - ); - try { - const fileContent = await NativeFile.readFile(item.path); - const backupNovel = JSON.parse(fileContent) as BackupNovel; - pluginIds.add(backupNovel.pluginId); - - const hasStoredCover = - backupNovel.cover && !backupNovel.cover.startsWith('http'); - if (hasStoredCover) { - backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; - } - - const novelMapping = await _restoreNovelAndChapters(backupNovel); - novelMappings.push(novelMapping); - novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); - - if (hasStoredCover) { - const coverBackupPath = coversDirPath + '/' + backupNovel.id; - if (await NativeFile.exists(coverBackupPath)) { - const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; - await NativeFile.mkdir(parentDirectory(coverPath)); - await NativeFile.copyFile(coverBackupPath, coverPath); - } - } - novelCount++; - } catch { - failedCount++; - } - } - } catch { - failedSectionCount++; - } - } - - // categories - if (manifest.sections.library) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringCategories'), - ); - } - const categoryFilePath = cacheDirPath + '/' + BackupEntryName.CATEGORY; - let categoryCount = 0; - let failedCategoryCount = 0; - - if (!manifest.sections.library) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(categoryFilePath))) { - failedSectionCount++; - } else { - try { - const fileContent = await NativeFile.readFile(categoryFilePath); - const categories: BackupCategory[] = JSON.parse(fileContent); - - for (const [index, category] of categories.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringCategoriesProgress', { - current: index + 1, - total: categories.length, - }), - ); - try { - await _restoreCategory( - { - ...category, - novelIds: category.novelIds.filter(novelId => - novelIdMap.has(novelId), - ), - }, - novelIdMap, - ); - categoryCount++; - } catch { - failedCategoryCount++; - } - } - } catch { - failedSectionCount++; - } - } - - // settings - if (manifest.sections.settings) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringSettings')); - } - const settingsFilePath = cacheDirPath + '/' + BackupEntryName.SETTING; - let settingsRestored = !manifest.sections.settings; - - if (!manifest.sections.settings) { - // Intentionally omitted from this backup. - } else if (!(await NativeFile.exists(settingsFilePath))) { - // Reported as a settings warning in the completion summary. - } else { - try { - const fileContent = await NativeFile.readFile(settingsFilePath); - const settingsData = JSON.parse(fileContent); - if (INSTALLED_PLUGINS_KEY in settingsData) { - pluginsFromSettings = parsePluginList( - settingsData[INSTALLED_PLUGINS_KEY], - ); - delete settingsData[INSTALLED_PLUGINS_KEY]; - } - restoreMMKVData(settingsData); - settingsRestored = true; - } catch { - // Included in the completion warning below. - } - } - - // installed plugin registry - if (manifest.sections.plugins) { - let restoredPlugins = pluginsFromSettings; - if (manifest.formatVersion === 2) { - const pluginMetadataPath = - cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; - if (!(await NativeFile.exists(pluginMetadataPath))) { - failedSectionCount++; - } else { - try { - restoredPlugins = parsePluginList( - await NativeFile.readFile(pluginMetadataPath), - ); - } catch { - failedSectionCount++; - } - } - } - const mergedPlugins = [ - ...new Map( - [...installedPluginsBeforeRestore, ...restoredPlugins].map(plugin => [ - plugin.id, - plugin, - ]), - ).values(), - ]; - MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); - } - - return { - novelCount, - failedNovelCount: failedCount, - categoryCount, - failedCategoryCount, - settingsRestored, - failedSectionCount, - pluginIds: [...pluginIds], - novelMappings, - manifest, - }; -}; diff --git a/test/mocks/nativeModules.js b/test/mocks/nativeModules.js index e5f48124bf..a228365d9d 100644 --- a/test/mocks/nativeModules.js +++ b/test/mocks/nativeModules.js @@ -81,6 +81,7 @@ jest.mock('@modules/native-zip-archive', () => ({ __esModule: true, default: { zip: jest.fn().mockResolvedValue(), + zipDirectories: jest.fn().mockResolvedValue(), unzip: jest.fn().mockResolvedValue(), remoteUnzip: jest.fn().mockResolvedValue(), remoteZip: jest.fn().mockResolvedValue(''),