From bbbcb85646dadd1dc0c947bdab86537af11d460a Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Wed, 11 Jan 2023 05:14:59 +0100 Subject: [PATCH 01/38] add clap, strip bom --- .gitignore | 3 + Cargo.lock | 1015 +++++++++++++++++++++++++++++------------------ Cargo.toml | 7 +- src/blockid.rs | 61 ++- src/database.rs | 33 +- src/main.rs | 78 ++-- src/stripbom.rs | 33 ++ 7 files changed, 790 insertions(+), 440 deletions(-) create mode 100644 src/stripbom.rs diff --git a/.gitignore b/.gitignore index be3a5c8..bbb1eb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /target **/*.rs.bk **/*.py + +/debug_*.json +/testout \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 4798eea..645efee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,1012 +1,1273 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler32" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" [[package]] name = "aho-corasick" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "aho-corasick" -version = "0.7.3" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ - "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr", ] [[package]] name = "ansi_term" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7", ] [[package]] name = "arrayvec" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" dependencies = [ - "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop", ] [[package]] name = "atty" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "termion", + "winapi 0.3.7", ] [[package]] name = "autocfg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" [[package]] name = "base64" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" dependencies = [ - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", ] [[package]] name = "bindgen" -version = "0.32.3" +version = "0.55.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443" dependencies = [ - "cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "clang-sys 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags", + "cexpr", + "cfg-if 0.1.7", + "clang-sys", + "clap 2.33.0", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2 1.0.49", + "quote 1.0.23", + "regex", + "rustc-hash", + "shlex", + "which", ] [[package]] name = "bitflags" -version = "1.0.4" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "byteorder" version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" [[package]] name = "bzip2" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" dependencies = [ - "bzip2-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "bzip2-sys", + "libc", ] [[package]] name = "bzip2-sys" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f" dependencies = [ - "cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "cc", + "libc", ] [[package]] name = "cc" version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5f3fee5eeb60324c2781f1e41286bdee933850fff9b3c672587fed5ec58c83" [[package]] name = "cexpr" -version = "0.2.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" dependencies = [ - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom", ] [[package]] name = "cfg-if" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" dependencies = [ - "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer", + "num-traits", + "time", ] [[package]] name = "clang-sys" -version = "0.21.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" dependencies = [ - "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "libloading 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "glob", + "libc", + "libloading", ] [[package]] name = "clap" version = "2.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "4.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ - "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim 0.10.0", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2 1.0.49", + "quote 1.0.23", + "syn 1.0.107", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", ] [[package]] name = "clicolors-control" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73abfd4c73d003a674ce5d2933fca6ce6c42480ea84a5ffe0a2dc39ed56300f9" dependencies = [ - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "atty", + "lazy_static", + "libc", + "winapi 0.3.7", ] [[package]] name = "cloudabi" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags", ] [[package]] name = "console" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf3720d3f3fc30b721ef1ae54e13af3264af4af39dc476a8de56a6ee1e2184b" dependencies = [ - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "clicolors-control 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "atty", + "clicolors-control", + "encode_unicode", + "lazy_static", + "libc", + "parking_lot", + "regex", + "termios", + "unicode-width", + "winapi 0.3.7", ] [[package]] name = "crc32fast" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7", ] [[package]] name = "crossbeam-deque" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" dependencies = [ - "crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" dependencies = [ - "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec", + "cfg-if 0.1.7", + "crossbeam-utils", + "lazy_static", + "memoffset", + "nodrop", + "scopeguard", ] [[package]] name = "crossbeam-utils" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.7", ] [[package]] name = "either" version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" [[package]] name = "encode_unicode" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd" [[package]] name = "env_logger" -version = "0.4.3" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" dependencies = [ - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "errno-dragonfly", + "libc", + "winapi 0.3.7", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", ] [[package]] name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "glob" -version = "0.2.11" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "indenter" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indicatif" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c60da1c9abea75996b70a931bba6c750730399005b61ccd853cee50ef3d0d0c" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "parking_lot", + "regex", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ - "console 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", ] [[package]] name = "itoa" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" [[package]] name = "kernel32-sys" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8", + "winapi-build", ] [[package]] name = "lazy_static" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.51" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libflate" version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c52384aeb22d0ce82a10d8ddf35f7fb4717d1b23eac5b94cd38d2050fb53766a" dependencies = [ - "adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "adler32", + "byteorder", + "crc32fast", ] [[package]] name = "libloading" -version = "0.4.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 1.0.0", + "winapi 0.3.7", ] [[package]] -name = "lock_api" -version = "0.1.5" +name = "linux-raw-sys" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] -name = "log" -version = "0.3.9" +name = "lock_api" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" dependencies = [ - "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "owning_ref", + "scopeguard", ] [[package]] name = "log" -version = "0.4.6" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 1.0.0", ] [[package]] name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "memchr" -version = "2.2.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" [[package]] name = "nodrop" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" [[package]] name = "nom" -version = "3.2.1" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr", + "version_check", ] [[package]] name = "num-integer" version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits", ] [[package]] name = "num-traits" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" [[package]] name = "num_cpus" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", ] [[package]] name = "number_prefix" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf9993e59c894e3c08aa1c2712914e9e6bf1fcbfc6bef283e2183df345a4fee" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits", ] [[package]] name = "numtoa" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" + +[[package]] +name = "once_cell" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" + +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "owning_ref" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" dependencies = [ - "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "stable_deref_trait", ] [[package]] name = "parking_lot" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" dependencies = [ - "lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api", + "parking_lot_core", ] [[package]] name = "parking_lot_core" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "rand", + "rustc_version", + "smallvec", + "winapi 0.3.7", ] +[[package]] +name = "paste" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" + [[package]] name = "pbr" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb73390ab68d81992bd994d145f697451bb0b54fd39738e72eef32458ad6907" dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys", + "libc", + "termion", + "time", + "winapi 0.2.8", ] [[package]] name = "peeking_take_while" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "podio" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" [[package]] -name = "proc-macro2" -version = "0.2.3" +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2 1.0.49", + "quote 1.0.23", + "syn 1.0.107", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.49", + "quote 1.0.23", + "version_check", ] [[package]] name = "proc-macro2" version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba92c84f814b3f9a44c5cfca7d2ad77fa10710867d2bbb1b3d175ab5f47daa12" dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid", ] [[package]] -name = "quote" -version = "0.4.2" +name = "proc-macro2" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ - "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-ident", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +dependencies = [ + "proc-macro2 0.4.28", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ - "proc-macro2 0.4.28 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.49", ] [[package]] name = "rand" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_jitter 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg", + "libc", + "rand_chacha", + "rand_core 0.4.0", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.7", ] [[package]] name = "rand_chacha" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg", + "rand_core 0.3.1", ] [[package]] name = "rand_core" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" dependencies = [ - "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0", ] [[package]] name = "rand_core" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" [[package]] name = "rand_hc" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rand_isaac" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rand_jitter" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "rand_core 0.4.0", + "winapi 0.3.7", ] [[package]] name = "rand_os" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.0", + "rdrand", + "winapi 0.3.7", ] [[package]] name = "rand_pcg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg", + "rand_core 0.4.0", ] [[package]] name = "rand_xorshift" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rayon" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473" dependencies = [ - "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque", + "either", + "rayon-core", ] [[package]] name = "rayon-core" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" dependencies = [ - "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque", + "lazy_static", + "libc", + "num_cpus", ] [[package]] name = "rdrand" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "redox_syscall" version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" [[package]] name = "redox_termios" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" dependencies = [ - "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall", ] [[package]] name = "regex" -version = "0.2.11" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ - "aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "regex" -version = "1.1.6" +name = "regex-syntax" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] -name = "regex-syntax" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" +name = "rust-duplicati-restore" +version = "0.0.2" dependencies = [ - "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "base64", + "chrono", + "clap 4.0.32", + "eyre", + "indicatif", + "num_cpus", + "pbr", + "rayon", + "serde", + "serde_json", + "serde_path_to_error", + "unqlite", + "zip", ] [[package]] -name = "regex-syntax" -version = "0.6.6" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] -name = "rust-duplicati-restore" -version = "0.0.2" +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "indicatif 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", - "unqlite 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "zip 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "semver", ] [[package]] -name = "rustc_version" -version = "0.2.3" +name = "rustix" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] name = "ryu" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" [[package]] name = "scopeguard" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" [[package]] name = "semver" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "semver-parser", ] [[package]] name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa5f7c20820475babd2c077c3ab5f8c77a31c15e16ea38687b4c02d3e48680f4" dependencies = [ - "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fc82bec244f168b23d1963b45c8bf5726e9a15a9d146a067f9081aeed2de79" dependencies = [ - "proc-macro2 0.4.28 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.32 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.28", + "quote 0.6.12", + "syn 0.15.32", ] [[package]] name = "serde_json" version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" dependencies = [ - "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa", + "ryu", + "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +dependencies = [ + "serde", +] + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + [[package]] name = "smallvec" version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" [[package]] name = "stable_deref_trait" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" [[package]] name = "strsim" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" version = "0.15.32" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "846620ec526c1599c070eff393bfeeeb88a93afa2513fc3b49f1fea84cf7b0ed" dependencies = [ - "proc-macro2 0.4.28 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.28", + "quote 0.6.12", + "unicode-xid", +] + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2 1.0.49", + "quote 1.0.23", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", ] [[package]] name = "termion" version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "numtoa", + "redox_syscall", + "redox_termios", ] [[package]] name = "termios" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", ] [[package]] name = "textwrap" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "thread_local" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width", ] [[package]] name = "time" version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "redox_syscall", + "winapi 0.3.7", ] [[package]] -name = "ucd-util" -version = "0.1.3" +name = "unicode-ident" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" [[package]] name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unqlite" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3a9e0f84985f05127d7f923ff129718a4db3585794e636540e109746dce452" dependencies = [ - "bindgen 0.32.3 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "bindgen", + "cc", + "libc", + "paste", ] [[package]] -name = "utf8-ranges" -version = "1.0.2" +name = "vec_map" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" [[package]] -name = "vec_map" -version = "0.8.1" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "which" -version = "1.0.5" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" dependencies = [ - "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", ] [[package]] name = "winapi" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] name = "winapi-build" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.7", +] [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" [[package]] name = "zip" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c18fc320faf909036e46ac785ea827f72e485304877faf1a3a39538d3714dbc3" dependencies = [ - "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libflate 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" -"checksum aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" -"checksum aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e6f484ae0c99fec2e858eb6134949117399f222608d84cadb3f58c1f97c2364c" -"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" -"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" -"checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" -"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" -"checksum bindgen 0.32.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8b242e11a8f446f5fc7b76b37e81d737cabca562a927bd33766dac55b5f1177f" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" -"checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" -"checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" -"checksum bzip2-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f" -"checksum cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5f3fee5eeb60324c2781f1e41286bdee933850fff9b3c672587fed5ec58c83" -"checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" -"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" -"checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" -"checksum clang-sys 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e414af9726e1d11660801e73ccc7fb81803fb5f49e5903a25b348b2b3b480d2e" -"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" -"checksum clicolors-control 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "73abfd4c73d003a674ce5d2933fca6ce6c42480ea84a5ffe0a2dc39ed56300f9" -"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum console 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2bf3720d3f3fc30b721ef1ae54e13af3264af4af39dc476a8de56a6ee1e2184b" -"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" -"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" -"checksum crossbeam-epoch 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" -"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" -"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" -"checksum encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd" -"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" -"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -"checksum indicatif 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c60da1c9abea75996b70a931bba6c750730399005b61ccd853cee50ef3d0d0c" -"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" -"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" -"checksum libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)" = "bedcc7a809076656486ffe045abeeac163da1b558e963a31e29fbfbeba916917" -"checksum libflate 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c52384aeb22d0ce82a10d8ddf35f7fb4717d1b23eac5b94cd38d2050fb53766a" -"checksum libloading 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fd38073de8f7965d0c17d30546d4bb6da311ab428d1c7a3fc71dff7f9d4979b9" -"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" -"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -"checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" -"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" -"checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" -"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" -"checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" -"checksum num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" -"checksum number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dbf9993e59c894e3c08aa1c2712914e9e6bf1fcbfc6bef283e2183df345a4fee" -"checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" -"checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" -"checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" -"checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" -"checksum pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "deb73390ab68d81992bd994d145f697451bb0b54fd39738e72eef32458ad6907" -"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" -"checksum podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" -"checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0" -"checksum proc-macro2 0.4.28 (registry+https://github.com/rust-lang/crates.io-index)" = "ba92c84f814b3f9a44c5cfca7d2ad77fa10710867d2bbb1b3d175ab5f47daa12" -"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408" -"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" -"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -"checksum rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" -"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -"checksum rand_jitter 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" -"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473" -"checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" -"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -"checksum redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)" = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" -"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -"checksum regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8f0a0bcab2fd7d1d7c54fa9eae6f43eddeb9ce2e7352f8518a814a4f65d60c58" -"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -"checksum regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dcfd8681eebe297b81d98498869d4aae052137651ad7b96822f09ceb690d0a96" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" -"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "aa5f7c20820475babd2c077c3ab5f8c77a31c15e16ea38687b4c02d3e48680f4" -"checksum serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)" = "58fc82bec244f168b23d1963b45c8bf5726e9a15a9d146a067f9081aeed2de79" -"checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" -"checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" -"checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" -"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -"checksum syn 0.15.32 (registry+https://github.com/rust-lang/crates.io-index)" = "846620ec526c1599c070eff393bfeeeb88a93afa2513fc3b49f1fea84cf7b0ed" -"checksum termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" -"checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" -"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" -"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" -"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" -"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" -"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" -"checksum unqlite 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "45db6e5b53a131a5eac62ca134f922351acf2bc1889d1d3be35a61d2ece70ea0" -"checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" -"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -"checksum which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" -"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" -"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum zip 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c18fc320faf909036e46ac785ea827f72e485304877faf1a3a39538d3714dbc3" + "bzip2", + "crc32fast", + "libflate", + "podio", + "time", +] diff --git a/Cargo.toml b/Cargo.toml index b389489..8079079 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,5 +15,8 @@ serde = {version = "1.0", features = ["derive"]} serde_json = "1.0.39" rayon = "1.0" num_cpus = "1.10.0" -unqlite = "1.4.1" -indicatif = "0.11.0" \ No newline at end of file +unqlite = "1.5" +indicatif = "0.11.0" +clap = { version = "4.0.32", features = ["derive"] } +eyre = "0.6.8" +serde_path_to_error = "0.1" diff --git a/src/blockid.rs b/src/blockid.rs index 486905a..d524f01 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -1,7 +1,13 @@ use crate::database::DB; +use crate::stripbom::StripBomBytes; use base64; +use eyre::Context; +use eyre::Result; use serde::Deserialize; use serde_json; +use serde_json::de::SliceRead; +use serde_json::Deserializer; + use std::fs; use std::fs::File; use std::io::prelude::*; @@ -25,7 +31,9 @@ pub enum FileType { #[derive(Debug)] pub struct FileEntry { path: String, + #[allow(unused)] metahash: String, + #[allow(unused)] metasize: i64, file_type: FileType, block_lists: Vec, @@ -76,11 +84,7 @@ impl FileEntry { } } - pub fn restore_file( - &self, - db: &DB, - restore_path: &str - ) { + pub fn restore_file(&self, db: &DB, restore_path: &str) { let root_path = Path::new(restore_path); let file_path = Path::new(&self.path[1..]); let path = Path::join(root_path, file_path); @@ -109,7 +113,7 @@ impl FileEntry { for (bi, hash) in binary_hashes.chunks(db.hash_size()).enumerate() { let hash = base64::encode(hash); let block = db.get_content_block(&hash); - + if let Some(block) = block { file.seek(SeekFrom::Start( (blockhashoffset + bi * db.block_size()) as u64, @@ -141,26 +145,57 @@ impl FileEntry { #[derive(Deserialize)] pub(self) struct IEntry { + #[serde(rename = "type")] + pub(self) filetype: String, + pub(self) path: String, pub(self) hash: Option, + pub(self) size: Option, + pub(self) metablockhash: Option, pub(self) metahash: String, pub(self) metasize: i64, - pub(self) path: String, - #[serde(rename = "type")] - pub(self) filetype: String, - pub(self) size: Option, + pub(self) time: Option, pub(self) blocklists: Option>, } /// Accepts the dlist as a string (must be read in first) /// Returns a Vec of FileEntrys -pub fn parse_dlist(dlist: &str) -> Vec { +pub fn parse_dlist(dlist: &[u8]) -> Result> { let mut file_entries = Vec::new(); - let entry_list: Vec = serde_json::from_str(dlist).unwrap(); + // { + // let debug_dlist = { + // let mut dfile = File::open("debug_dlist_pretty.json")?; + // let mut contents = String::new(); + // dfile.read_to_string(&mut contents)?; + // contents + // }; + // let read = SliceRead::new(debug_dlist.as_bytes()); + // let mut de = Deserializer::new(read); + // let obj: serde_json::Value = Deserialize::deserialize(&mut de).wrap_err("from_slice debug")?; + // //let obj: serde_json::Value = serde_json::from_slice(dlist) + + // let mut writer = Vec::new(); + // let mut ser = Serializer::with_formatter(&mut writer, PrettyFormatter::new()); + // let pretty = obj.serialize(&mut ser).wrap_err("to_string_pretty debug")?; + // let mut file = File::create("debug_dlist.json")?; + // file.write_all(&writer)?; + // } + + // { + // let mut file = File::create("debug_dlist.json")?; + // file.write_all(&dlist)?; + // } + + let read = SliceRead::new(dlist.strip_bom()); + let mut de = Deserializer::new(read); + let entry_list: Vec = + serde_path_to_error::deserialize(&mut de).wrap_err("deserialize entry_list")?; + for entry in entry_list { file_entries.push(FileEntry::from_ientry(&entry)); } - file_entries + Ok(file_entries) } + diff --git a/src/database.rs b/src/database.rs index 44b9ba4..3297d2a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -9,6 +9,8 @@ use std::io::Read; use std::path::Path; use unqlite::{Transaction, UnQLite, KV}; use zip; +use eyre::Result; +use eyre::eyre; #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future @@ -35,13 +37,15 @@ pub struct DB { } impl DB { - pub fn new(file: &str, manifest: &str) -> DB { + pub fn new(file: &str, manifest: &str) -> Result { let conn = UnQLite::create(file); - let manifest: Manifest = serde_json::from_str(manifest).unwrap(); - DB { conn, manifest } + conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; + let manifest: Manifest = serde_json::from_str(manifest)?; + let db = DB { conn, manifest }; + Ok(db) } - pub fn create_block_id_to_filenames(self, paths: &[String]) -> Self { + pub fn create_block_id_to_filenames(self, paths: &[String]) -> Result { // Iterate through dblocks, adding them to the db let pb = ProgressBar::new(paths.len() as u64); pb.set_style( @@ -54,23 +58,24 @@ impl DB { let conn = &self.conn; paths .par_iter() - .map(|path| { + .map(|zippath| { // In this stage, open the file - let file = File::open(&Path::new(path)).unwrap(); - let buf = BufReader::new(file); - let zip = zip::ZipArchive::new(buf).unwrap(); - (zip, path) + let zipfile = File::open(&Path::new(zippath)).unwrap(); + let zipbuf = BufReader::new(zipfile); + let zip = zip::ZipArchive::new(zipbuf).unwrap(); + (zip, zippath) }) - .map(|(mut zip, path)| { + .map(|(mut zip, zippath)| { // Convert to a list of paths let paths: Vec = (0..zip.len()) .map(|i| zip.by_index(i).unwrap().name().to_string()) .collect(); - (paths, path) + (paths, zippath) }) - .for_each(|(paths, path)| { - let bytes = path.as_bytes(); + .for_each(|(paths, zippath)| { + let bytes = zippath.as_bytes(); for p in paths { + println!("zippath:{} p:{}", zippath, p); let hash = base64::decode_config(&p, base64::URL_SAFE).unwrap(); conn.kv_store(hash, bytes).unwrap(); } @@ -78,7 +83,7 @@ impl DB { pb.inc(1); }); - self + Ok(self) } pub fn get_filename_from_block_id(&self, block_id: &str) -> Option { diff --git a/src/main.rs b/src/main.rs index ac3725a..0fc63c0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,11 @@ mod blockid; +mod stripbom; mod database; use blockid::*; +use clap::Parser; use database::*; +use eyre::{Context, Result}; use num_cpus; use pbr::ProgressBar; use rayon::prelude::*; @@ -13,38 +16,40 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use zip; +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct CliArgs { + /// the location of the backup + #[arg(short, long)] + backup_dir: String, + + /// a location to restore to + #[arg(short, long, value_name = "FILE")] + restore_dir: String, + + #[arg(short, long)] + cpu_count: Option, +} + fn main() { - println!("Enter the location of the backup:"); - let mut backup_dir = String::new(); - stdin() - .read_line(&mut backup_dir) - .expect("Did not enter a location."); - println!(); - let backup_dir = backup_dir.trim(); + let result = run(); + match result { + Err(err) => { + println!("err: {:?}", err); + } + Ok(_) => {} + } +} - println!("Enter a location to restore to:"); - let mut restore_dir = String::new(); - stdin() - .read_line(&mut restore_dir) - .expect("Did not enter a location."); - println!(); - let restore_dir = restore_dir.trim(); +fn run() -> Result<()> { + let args = CliArgs::parse(); + let backup_dir = args.backup_dir.trim(); + let restore_dir = args.restore_dir.trim(); - let db_location = Path::join(Path::new(backup_dir), Path::new("index.db")); + let db_location = Path::join(Path::new(restore_dir), Path::new("index.db")); let db_location = db_location.to_str().unwrap(); - println!( - "Enter number of threads to use (Default {}):", - num_cpus::get() - ); - let mut cpu_input = String::new(); - stdin() - .read_line(&mut cpu_input) - .expect("Did not enter a number"); - let cpu_count: usize = match cpu_input.trim().parse() { - Ok(i) => i, - Err(..) => num_cpus::get(), - }; + let cpu_count: usize = args.cpu_count.unwrap_or_else(|| num_cpus::get()); println!(); // Set CPU count @@ -69,12 +74,15 @@ fn main() { println!("Parsing dlist"); // Open dlist file - let mut dlist_zip = zip::ZipArchive::new(File::open(dlist.clone()).unwrap()).unwrap(); - let mut dlist_file = dlist_zip.by_name("filelist.json").unwrap(); - let mut dlist_contents = String::new(); - dlist_file.read_to_string(&mut dlist_contents).unwrap(); - let file_entries = parse_dlist(&dlist_contents); - + let dlist_reader = File::open(dlist.clone()).wrap_err_with(|| format!("open {}", dlist))?; + let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; + let mut dlist_file = dlist_zip.by_name("filelist.json")?; + let mut dlist_contents = Vec::new(); + dlist_file.read_to_end(&mut dlist_contents)?; + let file_entries = + parse_dlist(&dlist_contents).wrap_err_with(|| format!("parse_dlist {}", dlist))?; + + println!("Parsing manifest"); // Open Manifest let mut manifest_zip = zip::ZipArchive::new(File::open(dlist.clone()).unwrap()).unwrap(); let mut manifest_file = manifest_zip.by_name("manifest").unwrap(); @@ -105,7 +113,7 @@ fn main() { println!(); println!("Indexing dblocks"); let dblock_db = - DB::new(db_location, &manifest_contents).create_block_id_to_filenames(&zip_file_names); + DB::new(db_location, &manifest_contents)?.create_block_id_to_filenames(&zip_file_names)?; println!("Restoring directory structure"); let mut pb = ProgressBar::new(folder_count as u64); @@ -124,4 +132,6 @@ fn main() { f.restore_file(&dblock_db, &restore_dir); pb.lock().unwrap().inc(); }); + + Ok(()) } diff --git a/src/stripbom.rs b/src/stripbom.rs new file mode 100644 index 0000000..46f57a6 --- /dev/null +++ b/src/stripbom.rs @@ -0,0 +1,33 @@ +pub trait StripBom { + fn strip_bom(&self) -> &str; +} + +impl StripBom for str { + fn strip_bom(&self) -> &str { + if self.starts_with("\u{feff}") { + &self[3..] + } else { + &self[..] + } + } +} + +impl StripBom for String { + fn strip_bom(&self) -> &str { + &self[..].strip_bom() + } +} + +pub trait StripBomBytes { + fn strip_bom(&self) -> &[u8]; +} + +impl StripBomBytes for [u8] { + fn strip_bom<'a>(&'a self) -> &'a [u8] { + if self.starts_with(&[0xEF, 0xBB, 0xBF]) { + &self[3..] + } else { + &self[..] + } + } +} From 1e30a34a010dde554bb6df9c1eb154e0a2f8ce09 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Wed, 11 Jan 2023 07:22:32 +0100 Subject: [PATCH 02/38] parallel zip --- .gitignore | 3 +- Cargo.lock | 709 ++++++++++++++++++++++++++++++------------------ Cargo.toml | 11 +- src/blockid.rs | 1 - src/database.rs | 243 ++++++++++++++--- src/main.rs | 60 ++-- 6 files changed, 697 insertions(+), 330 deletions(-) diff --git a/.gitignore b/.gitignore index bbb1eb1..3bf60d2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ **/*.py /debug_*.json -/testout \ No newline at end of file +/testout +/dhat-heap.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 645efee..624cece 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,10 +3,31 @@ version = 3 [[package]] -name = "adler32" -version = "1.0.3" +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", + "opaque-debug", +] [[package]] name = "aho-corasick" @@ -48,9 +69,24 @@ dependencies = [ [[package]] name = "autocfg" -version = "0.1.2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" @@ -61,6 +97,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "base64ct" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" + [[package]] name = "bindgen" version = "0.55.1" @@ -77,8 +119,8 @@ dependencies = [ "lazycell", "log", "peeking_take_while", - "proc-macro2 1.0.49", - "quote 1.0.23", + "proc-macro2", + "quote", "regex", "rustc-hash", "shlex", @@ -91,17 +133,26 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + [[package]] name = "byteorder" -version = "1.3.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bzip2" -version = "0.3.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ "bzip2-sys", "libc", @@ -109,19 +160,23 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.7" +version = "0.1.11+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" dependencies = [ "cc", "libc", + "pkg-config", ] [[package]] name = "cc" -version = "1.0.35" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5f3fee5eeb60324c2781f1e41286bdee933850fff9b3c672587fed5ec58c83" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +dependencies = [ + "jobserver", +] [[package]] name = "cexpr" @@ -152,7 +207,16 @@ checksum = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" dependencies = [ "num-integer", "num-traits", - "time", + "time 0.1.42", +] + +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", ] [[package]] @@ -204,9 +268,9 @@ checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.49", - "quote 1.0.23", - "syn 1.0.107", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -230,15 +294,6 @@ dependencies = [ "winapi 0.3.7", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "console" version = "0.7.5" @@ -257,13 +312,38 @@ dependencies = [ "winapi 0.3.7", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" -version = "1.2.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 0.1.7", + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.14", ] [[package]] @@ -273,7 +353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" dependencies = [ "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.2.2", ] [[package]] @@ -284,11 +364,11 @@ checksum = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" dependencies = [ "arrayvec", "cfg-if 0.1.7", - "crossbeam-utils", + "crossbeam-utils 0.2.2", "lazy_static", "memoffset", "nodrop", - "scopeguard", + "scopeguard 0.3.3", ] [[package]] @@ -300,6 +380,52 @@ dependencies = [ "cfg-if 0.1.7", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "dhat" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3" +dependencies = [ + "backtrace", + "lazy_static", + "mintex", + "parking_lot", + "rustc-hash", + "serde", + "serde_json", + "thousands", +] + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "either" version = "1.5.2" @@ -357,10 +483,30 @@ dependencies = [ ] [[package]] -name = "fuchsia-cprng" -version = "0.1.1" +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "generic-array" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gimli" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" [[package]] name = "glob" @@ -383,6 +529,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "humantime" version = "1.3.0" @@ -439,6 +594,21 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -451,9 +621,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lazycell" @@ -467,17 +637,6 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" -[[package]] -name = "libflate" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52384aeb22d0ce82a10d8ddf35f7fb4717d1b23eac5b94cd38d2050fb53766a" -dependencies = [ - "adler32", - "byteorder", - "crc32fast", -] - [[package]] name = "libloading" version = "0.7.4" @@ -496,12 +655,12 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" -version = "0.1.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ - "owning_ref", - "scopeguard", + "autocfg", + "scopeguard 1.1.0", ] [[package]] @@ -525,6 +684,25 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "mintex" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb" +dependencies = [ + "once_cell", + "sys-info", +] + [[package]] name = "nodrop" version = "0.1.13" @@ -580,6 +758,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +[[package]] +name = "object" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d864c91689fdc196779b98dba0aceac6118594c2df6ee5d943eb6a8df4d107a" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.17.0" @@ -587,25 +774,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] -name = "os_str_bytes" -version = "6.4.1" +name = "opaque-debug" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] -name = "owning_ref" -version = "0.4.0" +name = "os_str_bytes" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" -dependencies = [ - "stable_deref_trait", -] +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "parking_lot" -version = "0.7.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core", @@ -613,15 +797,26 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.4.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ + "cfg-if 1.0.0", "libc", - "rand", - "rustc_version", + "redox_syscall 0.2.16", "smallvec", - "winapi 0.3.7", + "windows-sys", +] + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", ] [[package]] @@ -630,6 +825,18 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "pbr" version = "1.0.1" @@ -639,7 +846,7 @@ dependencies = [ "kernel32-sys", "libc", "termion", - "time", + "time 0.1.42", "winapi 0.2.8", ] @@ -650,10 +857,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] -name = "podio" -version = "0.1.6" +name = "pkg-config" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro-error" @@ -662,9 +869,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.49", - "quote 1.0.23", - "syn 1.0.107", + "proc-macro2", + "quote", + "syn", "version_check", ] @@ -674,20 +881,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.49", - "quote 1.0.23", + "proc-macro2", + "quote", "version_check", ] -[[package]] -name = "proc-macro2" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92c84f814b3f9a44c5cfca7d2ad77fa10710867d2bbb1b3d175ab5f47daa12" -dependencies = [ - "unicode-xid", -] - [[package]] name = "proc-macro2" version = "1.0.49" @@ -703,129 +901,20 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quote" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" -dependencies = [ - "proc-macro2 0.4.28", -] - [[package]] name = "quote" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ - "proc-macro2 1.0.49", -] - -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg", - "libc", - "rand_chacha", - "rand_core 0.4.0", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi 0.3.7", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg", - "rand_core 0.3.1", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.0", + "proc-macro2", ] [[package]] name = "rand_core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" - -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b9ea758282efe12823e0d952ddb269d2e1897227e464919a554f2a03ef1b832" -dependencies = [ - "libc", - "rand_core 0.4.0", - "winapi 0.3.7", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.0", - "rdrand", - "winapi 0.3.7", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg", - "rand_core 0.4.0", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "rayon" @@ -851,19 +940,19 @@ dependencies = [ ] [[package]] -name = "rdrand" -version = "0.4.0" +name = "redox_syscall" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] +checksum = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" [[package]] name = "redox_syscall" -version = "0.1.54" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] [[package]] name = "redox_termios" @@ -871,7 +960,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" dependencies = [ - "redox_syscall", + "redox_syscall 0.1.54", ] [[package]] @@ -898,6 +987,8 @@ dependencies = [ "base64", "chrono", "clap 4.0.32", + "crossbeam-channel", + "dhat", "eyre", "indicatif", "num_cpus", @@ -906,24 +997,22 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", + "smallvec", "unqlite", "zip", ] [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "rustc-demangle" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] -name = "rustc_version" -version = "0.2.3" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" @@ -952,38 +1041,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" [[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" +name = "scopeguard" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.90" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5f7c20820475babd2c077c3ab5f8c77a31c15e16ea38687b4c02d3e48680f4" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.90" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fc82bec244f168b23d1963b45c8bf5726e9a15a9d146a067f9081aeed2de79" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ - "proc-macro2 0.4.28", - "quote 0.6.12", - "syn 0.15.32", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -992,7 +1072,7 @@ version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" dependencies = [ - "itoa", + "itoa 0.4.3", "ryu", "serde", ] @@ -1006,6 +1086,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "0.1.1" @@ -1014,15 +1116,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "smallvec" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" - -[[package]] -name = "stable_deref_trait" -version = "1.1.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "strsim" @@ -1037,15 +1133,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "syn" -version = "0.15.32" +name = "subtle" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "846620ec526c1599c070eff393bfeeeb88a93afa2513fc3b49f1fea84cf7b0ed" -dependencies = [ - "proc-macro2 0.4.28", - "quote 0.6.12", - "unicode-xid", -] +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" @@ -1053,11 +1144,21 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ - "proc-macro2 1.0.49", - "quote 1.0.23", + "proc-macro2", + "quote", "unicode-ident", ] +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -1075,7 +1176,7 @@ checksum = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" dependencies = [ "libc", "numtoa", - "redox_syscall", + "redox_syscall 0.1.54", "redox_termios", ] @@ -1097,6 +1198,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "time" version = "0.1.42" @@ -1104,10 +1211,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ "libc", - "redox_syscall", + "redox_syscall 0.1.54", "winapi 0.3.7", ] +[[package]] +name = "time" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "itoa 1.0.5", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + [[package]] name = "unicode-ident" version = "1.0.6" @@ -1120,12 +1260,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unqlite" version = "1.5.0" @@ -1261,13 +1395,50 @@ checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" [[package]] name = "zip" -version = "0.5.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18fc320faf909036e46ac785ea827f72e485304877faf1a3a39538d3714dbc3" +checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" dependencies = [ + "aes", + "byteorder", "bzip2", + "constant_time_eq", "crc32fast", - "libflate", - "podio", - "time", + "crossbeam-utils 0.8.14", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time 0.3.17", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.5+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596" +dependencies = [ + "cc", + "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 8079079..21a8d64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,14 @@ version = "0.0.2" authors = ["Nathan McCarty "] edition = "2018" + +[features] +dhat-heap = [] # if you are doing heap profiling +dhat-ad-hoc = [] # if you are doing ad hoc profiling + [dependencies] -zip = "0.5.2" +#zip = { version = "*", path = "../zip" } +zip = "*" chrono = "0.4.0" base64 = "0.10.1" pbr = "1.0.1" @@ -20,3 +26,6 @@ indicatif = "0.11.0" clap = { version = "4.0.32", features = ["derive"] } eyre = "0.6.8" serde_path_to_error = "0.1" +smallvec = "*" +crossbeam-channel = "*" +dhat = "*" diff --git a/src/blockid.rs b/src/blockid.rs index d524f01..5a6d11f 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -198,4 +198,3 @@ pub fn parse_dlist(dlist: &[u8]) -> Result> { Ok(file_entries) } - diff --git a/src/database.rs b/src/database.rs index 3297d2a..b481d81 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,16 +1,25 @@ use base64; +use eyre::eyre; +use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::*; use serde::Deserialize; use serde_json; +use smallvec::SmallVec; +use std::collections::HashMap; use std::fs::File; +use std::hash::Hash; use std::io::BufReader; +use std::io::IoSliceMut; use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use unqlite::{Transaction, UnQLite, KV}; use zip; -use eyre::Result; -use eyre::eyre; #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future @@ -31,17 +40,36 @@ struct Manifest { pub(self) app_version: String, } +pub struct ZipFilePath { + path: String, +} + +pub struct HashToPath { + hash2path: HashMap, Arc>, +} + +impl HashToPath { + pub fn new() -> Self { + let hash2path = HashMap::new(); + + Self { hash2path } + } +} + pub struct DB { - conn: UnQLite, + // conn: UnQLite, + inner: Arc>, manifest: Manifest, } impl DB { pub fn new(file: &str, manifest: &str) -> Result { - let conn = UnQLite::create(file); - conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; + // let conn = UnQLite::create(file); + // conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; let manifest: Manifest = serde_json::from_str(manifest)?; - let db = DB { conn, manifest }; + + let inner = Arc::new(Mutex::new(HashToPath::new())); + let db = DB { inner, manifest }; Ok(db) } @@ -55,47 +83,119 @@ impl DB { ) .progress_chars("##-"), ); - let conn = &self.conn; - paths - .par_iter() - .map(|zippath| { - // In this stage, open the file - let zipfile = File::open(&Path::new(zippath)).unwrap(); - let zipbuf = BufReader::new(zipfile); - let zip = zip::ZipArchive::new(zipbuf).unwrap(); - (zip, zippath) - }) - .map(|(mut zip, zippath)| { - // Convert to a list of paths - let paths: Vec = (0..zip.len()) - .map(|i| zip.by_index(i).unwrap().name().to_string()) - .collect(); - (paths, zippath) - }) - .for_each(|(paths, zippath)| { - let bytes = zippath.as_bytes(); - for p in paths { - println!("zippath:{} p:{}", zippath, p); - let hash = base64::decode_config(&p, base64::URL_SAFE).unwrap(); - conn.kv_store(hash, bytes).unwrap(); - } - conn.commit().unwrap(); - pb.inc(1); - }); + //let inner = &self.inner; + for zippath in paths { + // In this stage, open the file + + let zipbuf = MyCloneFileReader::new(Path::new(&zippath).to_path_buf())?; + let zip = zip::ZipArchive::new(zipbuf).unwrap(); + + let zip_len = zip.len(); + + //for zipfile in zip.file_names() + + // Convert to a list of paths + + let mut hvec = Vec::new(); + let (sender, receiver) = crossbeam_channel::unbounded(); + + // make workers + for t in 0..4 { + println!("Make worker {}", t); + + let receiver = receiver.clone(); // clone for this thread + + let azippath = Arc::new(ZipFilePath { + path: zippath.clone(), + }); + let zippath = zippath.clone(); + let inner = self.inner.clone(); + let mut zip = zip.clone(); + let handler = std::thread::spawn(move || { + //let mut rng = rand::thread_rng(); // each thread have one + // let zipfile = File::open(&Path::new(&zippath)).unwrap(); + // let zipbuf = BufReader::with_capacity(1 * 1024, zipfile); + // let mut zip = zip::ZipArchive::new(zipbuf).unwrap(); + + loop { + let r = receiver.recv(); + match r { + Ok(file_index) => { + //let s = rng.gen_range(100..1000); + + //thread::sleep(Duration::from_millis(s)); + + let hash_path = + zip.by_index(file_index).unwrap().name().to_string(); + let hash = + base64::decode_config(&hash_path, base64::URL_SAFE).unwrap(); + { + let mut inner = inner.lock().unwrap(); + inner.hash2path.insert(hash.into(), azippath.clone()); + } + } + _ => break, + } + } + }); + + hvec.push(handler); + } + + for i in 0..zip_len { + sender.send(i).unwrap(); + } + drop(sender); + // let paths: Vec = (0..zip_len) + // .into_par_iter() + // .map_init( + // || { + + // zip + // }, + // |zip, i| , + // ) + // .collect(); + + // let bytes = zippath.as_bytes(); + // for p in paths { + + // //conn.kv_store(hash, bytes).unwrap(); + + // //println!("len: {}", hash.len()); + // //println!("zippath:{} hash:{}", zippath, p); + + // } + //conn.commit().unwrap(); + for h in hvec { + h.join().unwrap(); + } + + pb.inc(1); + } Ok(self) } pub fn get_filename_from_block_id(&self, block_id: &str) -> Option { - let conn = &self.conn; + //let conn = &self.conn; // println!("{}", block_id); // let converted_block_id = base64_url_to_plain(block_id); - let result = conn.kv_fetch(base64::decode_config(block_id, base64::STANDARD).unwrap()); - if let Ok(path_bytes) = result { - Some(String::from_utf8(path_bytes).unwrap()) - } else { - None - } + let key = base64::decode_config(block_id, base64::STANDARD).unwrap(); + let key = SmallVec::from(key); + + self.inner + .lock() + .unwrap() + .hash2path + .get(&key) + .map(|v| v.path.clone()) + // let result = conn.kv_fetch(key); + // if let Ok(path_bytes) = result { + // Some(String::from_utf8(path_bytes).unwrap()) + // } else { + // None + // } } pub fn get_content_block(&self, block_id: &str) -> Option> { @@ -130,3 +230,66 @@ impl DB { 32 } } + +struct MyCloneFileReader { + path: PathBuf, + buf_reader: BufReader, +} + +impl Clone for MyCloneFileReader { + fn clone(&self) -> Self { + Self::new(self.path.clone()).unwrap() + } +} + +impl MyCloneFileReader { + pub fn new(path: PathBuf) -> Result { + let target_file = File::open(&path)?; + let filebuf = BufReader::with_capacity(2 * 1024, target_file); + + Ok(Self { + path, + buf_reader: filebuf, + }) + } +} + +impl Read for MyCloneFileReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.buf_reader.read(buf) + } + + // fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> std::io::Result<()> { + // self.buf_reader.read_buf(buf) + // } + + fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { + self.buf_reader.read_exact(buf) + } + + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> std::io::Result { + self.buf_reader.read_vectored(bufs) + } + + // fn is_read_vectored(&self) -> bool { + // self.buf_reader.is_read_vectored() + // } + + fn read_to_end(&mut self, buf: &mut Vec) -> std::io::Result { + self.buf_reader.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> std::io::Result { + self.buf_reader.read_to_string(buf) + } +} + +impl Seek for MyCloneFileReader { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.buf_reader.seek(pos) + } + + fn stream_position(&mut self) -> std::io::Result { + self.buf_reader.stream_position() + } +} diff --git a/src/main.rs b/src/main.rs index 0fc63c0..48eb2d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,25 @@ mod blockid; -mod stripbom; mod database; +mod stripbom; use blockid::*; +use chrono::Duration; use clap::Parser; use database::*; +use eyre::eyre; use eyre::{Context, Result}; use num_cpus; use pbr::ProgressBar; use rayon::prelude::*; use std::fs; use std::fs::File; -use std::io::{stdin, Read}; +use std::io::Read; use std::path::Path; use std::sync::{Arc, Mutex}; use zip; +use crate::stripbom::StripBom; + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct CliArgs { @@ -32,6 +36,14 @@ struct CliArgs { } fn main() { + #[cfg(feature = "dhat-heap")] + std::thread::spawn(|| { + let _profiler = dhat::Profiler::new_heap(); + + std::thread::sleep(std::time::Duration::from_secs(60)); + }); + + std::thread::sleep(std::time::Duration::from_millis(100)); let result = run(); match result { Err(err) => { @@ -68,30 +80,38 @@ fn run() -> Result<()> { dlist_file_names.sort(); - let dlist = dlist_file_names[dlist_file_names.len() - 1].clone(); + let dlist = dlist_file_names + .last() + .ok_or_else(|| eyre!("last modified dlist file not found"))?; println!("{} appears to be newest dlist, using it.", dlist); println!("Parsing dlist"); // Open dlist file - let dlist_reader = File::open(dlist.clone()).wrap_err_with(|| format!("open {}", dlist))?; - let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; - let mut dlist_file = dlist_zip.by_name("filelist.json")?; - let mut dlist_contents = Vec::new(); - dlist_file.read_to_end(&mut dlist_contents)?; - let file_entries = - parse_dlist(&dlist_contents).wrap_err_with(|| format!("parse_dlist {}", dlist))?; + let file_entries = { + let dlist_reader = File::open(dlist.clone()).wrap_err_with(|| format!("open {}", dlist))?; + let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; + let mut dlist_file = dlist_zip.by_name("filelist.json")?; + let mut dlist_contents = Vec::new(); + dlist_file.read_to_end(&mut dlist_contents)?; + + parse_dlist(&dlist_contents).wrap_err_with(|| format!("parse_dlist {}", dlist))? + }; println!("Parsing manifest"); // Open Manifest - let mut manifest_zip = zip::ZipArchive::new(File::open(dlist.clone()).unwrap()).unwrap(); - let mut manifest_file = manifest_zip.by_name("manifest").unwrap(); - let mut manifest_contents = String::new(); - manifest_file - .read_to_string(&mut manifest_contents) - .unwrap(); - let manifest_contents = manifest_contents.replace("\u{feff}", ""); - let manifest_contents = manifest_contents.trim(); + let manifest_contents = { + let manifest_file = File::open(dlist.clone())?; + let mut manifest_zip = zip::ZipArchive::new(manifest_file)?; + let mut manifest_file = manifest_zip.by_name("manifest")?; + let mut manifest_contents = String::new(); + manifest_file + .read_to_string(&mut manifest_contents) + .wrap_err("read manifest")?; + let manifest_contents = manifest_contents.strip_bom(); + let manifest_contents = manifest_contents.trim(); + manifest_contents.to_owned() + }; let file_count = file_entries.iter().filter(|f| f.is_file()).count(); println!("{} files to be restored", file_count); @@ -135,3 +155,7 @@ fn run() -> Result<()> { Ok(()) } + +#[cfg(feature = "dhat-heap")] +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; From cde3fc9be11f03ba8d1a6994ef7a0e60a7ee5793 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:34:40 +0100 Subject: [PATCH 03/38] sorting works --- .gitignore | 3 +- Cargo.lock | 2 - Cargo.toml | 4 +- src/blockid.rs | 175 ++++++++++++++++++++----------- src/database.rs | 273 ++++++++++++++++++++++++++++++++++-------------- src/main.rs | 162 +++++++++++++++++++--------- 6 files changed, 424 insertions(+), 195 deletions(-) diff --git a/.gitignore b/.gitignore index 3bf60d2..0821766 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ /debug_*.json /testout -/dhat-heap.json \ No newline at end of file +/dhat-heap.json +.vscode/settings.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 624cece..1bb9d45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,8 +1396,6 @@ checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" [[package]] name = "zip" version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" dependencies = [ "aes", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 21a8d64..b819f97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,8 @@ dhat-heap = [] # if you are doing heap profiling dhat-ad-hoc = [] # if you are doing ad hoc profiling [dependencies] -#zip = { version = "*", path = "../zip" } -zip = "*" +zip = { version = "*", path = "../zip" } +#zip = "*" chrono = "0.4.0" base64 = "0.10.1" pbr = "1.0.1" diff --git a/src/blockid.rs b/src/blockid.rs index 5a6d11f..3b44cf3 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -1,3 +1,4 @@ +use crate::database::BlockLocation; use crate::database::DB; use crate::stripbom::StripBomBytes; use base64; @@ -8,6 +9,8 @@ use serde_json; use serde_json::de::SliceRead; use serde_json::Deserializer; +use eyre::eyre; +use std::cmp::Ordering; use std::fs; use std::fs::File; use std::io::prelude::*; @@ -15,7 +18,7 @@ use std::io::SeekFrom; use std::io::Write; use std::path::Path; -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] pub enum FileType { File { hash: String, @@ -28,7 +31,31 @@ pub enum FileType { SymLink, } -#[derive(Debug)] +impl FileType { + pub fn is_file(&self) -> bool { + match self { + FileType::File { .. } => true, + _ => false, + } + } + + #[allow(unused)] + pub fn is_nonzero_file(&self) -> bool { + match self { + FileType::File { size, .. } => *size > 0, + _ => false, + } + } + + pub fn is_folder(&self) -> bool { + match self { + FileType::Folder { .. } => true, + _ => false, + } + } +} + +#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct FileEntry { path: String, #[allow(unused)] @@ -40,7 +67,7 @@ pub struct FileEntry { } impl FileEntry { - pub(self) fn from_ientry(ientry: &IEntry) -> FileEntry { + pub(self) fn from_ientry(ientry: &IEntry) -> Result { let path = ientry.path.clone(); let metahash = ientry.metahash.clone(); let metasize = ientry.metasize; @@ -51,80 +78,129 @@ impl FileEntry { }; let file_type = match ientry.filetype.as_ref() { "File" => FileType::File { - hash: ientry.hash.clone().unwrap(), - size: ientry.size.unwrap(), - time: ientry.time.clone().unwrap(), + hash: ientry.hash.clone().ok_or_else(|| eyre!("hash not found"))?, + size: ientry.size.clone().ok_or_else(|| eyre!("size not found"))?, + time: ientry.time.clone().ok_or_else(|| eyre!("time not found"))?, }, "Folder" => FileType::Folder { - metablockhash: ientry.metablockhash.clone().unwrap(), + metablockhash: ientry + .metablockhash + .clone() + .ok_or_else(|| eyre!("metablockhash not found"))?, }, _ => FileType::SymLink, }; - FileEntry { + Ok(FileEntry { path, metahash, metasize, file_type, block_lists, - } + }) } pub fn is_file(&self) -> bool { - match self.file_type { - FileType::File { .. } => true, - _ => false, - } + self.file_type.is_file() } pub fn is_folder(&self) -> bool { - match self.file_type { - FileType::Folder { .. } => true, - _ => false, + self.file_type.is_folder() + } + + /// Optional. Used for sorting. + pub fn get_first_bytes_location(&self, db: &DB) -> Option { + match &self.file_type { + FileType::File { hash, .. } => { + if self.block_lists.is_empty() { + db.get_block_id_location(&hash) + } else { + let first = self.block_lists.first(); + + first.map(|bid| db.get_block_id_location(bid)).flatten() + } + } + _ => None, } } - pub fn restore_file(&self, db: &DB, restore_path: &str) { + /// Optional. Used for sorting. + pub fn compare(&self, othr: &FileEntry, db: &DB) -> Ordering { + let a = self.get_first_bytes_location(db); + let b = othr.get_first_bytes_location(db); + + a.cmp(&b) + } + + pub fn restore_file(&self, db: &DB, restore_path: &str) -> Result<()> { let root_path = Path::new(restore_path); - let file_path = Path::new(&self.path[1..]); - let path = Path::join(root_path, file_path); + let dfile_path = &self.path[0..]; + let dfile_path = dfile_path.replacen(":\\", "\\", 1); + let dfile_path = dfile_path.replace("\\", "/"); + let relative_file_path = Path::new(&dfile_path); + + let path = Path::join(root_path, relative_file_path); match &self.file_type { FileType::Folder { .. } => { - fs::create_dir_all(path).unwrap(); + fs::create_dir_all(path)?; } FileType::File { hash, size, .. } => { // Small files only have one block if self.block_lists.is_empty() { - let mut file = File::create(path.clone()).unwrap(); - let block = db.get_content_block(hash); + let loc = db.get_block_id_location(hash); + println!( + "restoring file {:?}, index:{:?}", + relative_file_path, + loc.map(|loc| loc.file_index) + ); + + let mut out_file = File::create(path.clone())?; + let block = db.get_content_block(hash)?; if let Some(block) = block { - file.write_all(block.as_ref()).unwrap(); + out_file + .write_all(block.as_ref()) + .wrap_err("write single-block file")?; } else if *size > 0 { - println!("Missing block {} for {}", hash, path.to_str().unwrap()); + println!( + "Missing block {} for {}", + hash, + path.to_str().unwrap_or("not utf8?") + ); } } else { - let mut file = File::create(path.clone()).unwrap(); + let loc = self + .block_lists + .first() + .map(|hash| db.get_block_id_location(hash)) + .flatten(); + println!( + "restoring file {:?}, index:{:?}", + relative_file_path, + loc.map(|loc| loc.file_index) + ); + let mut out_file = File::create(path.clone())?; // Each blockid points to a list of blockids for (blhi, blh) in self.block_lists.iter().enumerate() { let blockhashoffset = blhi * db.offset_size(); - let binary_hashes = db.get_content_block(blh); + let binary_hashes = db.get_content_block(blh)?; if let Some(binary_hashes) = binary_hashes { - for (bi, hash) in binary_hashes.chunks(db.hash_size()).enumerate() { - let hash = base64::encode(hash); - let block = db.get_content_block(&hash); + for (bi, bhash) in binary_hashes.chunks(db.hash_size()).enumerate() { + let bhash = base64::encode(bhash); + let block = db.get_content_block(&bhash)?; if let Some(block) = block { - file.seek(SeekFrom::Start( - (blockhashoffset + bi * db.block_size()) as u64, - )) - .unwrap(); - file.write_all(&block).unwrap(); + out_file + .seek(SeekFrom::Start( + (blockhashoffset + bi * db.block_size()) as u64, + )) + .wrap_err("seek blockhashoffset + bi * db.block_size()")?; + out_file.write_all(&block).wrap_err("write block")?; } else { println!( "Failed to find block {} for {}", - hash, - path.to_str().unwrap() + bhash, + path.to_str().unwrap_or("not utf8?") ); } } @@ -140,6 +216,7 @@ impl FileEntry { } _ => (), } + Ok(()) } } @@ -163,29 +240,6 @@ pub(self) struct IEntry { /// Returns a Vec of FileEntrys pub fn parse_dlist(dlist: &[u8]) -> Result> { let mut file_entries = Vec::new(); - // { - // let debug_dlist = { - // let mut dfile = File::open("debug_dlist_pretty.json")?; - // let mut contents = String::new(); - // dfile.read_to_string(&mut contents)?; - // contents - // }; - // let read = SliceRead::new(debug_dlist.as_bytes()); - // let mut de = Deserializer::new(read); - // let obj: serde_json::Value = Deserialize::deserialize(&mut de).wrap_err("from_slice debug")?; - // //let obj: serde_json::Value = serde_json::from_slice(dlist) - - // let mut writer = Vec::new(); - // let mut ser = Serializer::with_formatter(&mut writer, PrettyFormatter::new()); - // let pretty = obj.serialize(&mut ser).wrap_err("to_string_pretty debug")?; - // let mut file = File::create("debug_dlist.json")?; - // file.write_all(&writer)?; - // } - - // { - // let mut file = File::create("debug_dlist.json")?; - // file.write_all(&dlist)?; - // } let read = SliceRead::new(dlist.strip_bom()); let mut de = Deserializer::new(read); @@ -193,7 +247,8 @@ pub fn parse_dlist(dlist: &[u8]) -> Result> { serde_path_to_error::deserialize(&mut de).wrap_err("deserialize entry_list")?; for entry in entry_list { - file_entries.push(FileEntry::from_ientry(&entry)); + let entry = FileEntry::from_ientry(&entry).wrap_err("FileEntry::from_ientry")?; + file_entries.push(entry); } Ok(file_entries) diff --git a/src/database.rs b/src/database.rs index b481d81..cb1dd9b 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,14 +1,12 @@ use base64; -use eyre::eyre; use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; -use rayon::prelude::*; use serde::Deserialize; use serde_json; use smallvec::SmallVec; use std::collections::HashMap; use std::fs::File; -use std::hash::Hash; +use std::io::BufRead; use std::io::BufReader; use std::io::IoSliceMut; use std::io::Read; @@ -16,10 +14,11 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; use std::path::PathBuf; +use std::sync::atomic::AtomicU32; use std::sync::Arc; use std::sync::Mutex; -use unqlite::{Transaction, UnQLite, KV}; use zip; +use zip::ZipArchive; #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future @@ -40,19 +39,48 @@ struct Manifest { pub(self) app_version: String, } -pub struct ZipFilePath { - path: String, +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +/// Path to dblock.zip +pub struct ZipLocation { + pub path_str: String, + pub path: PathBuf, } +#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] +pub struct BlockLocation { + /// Which file inside the zip + pub file_index: usize, + + /// Which dblock.zip file + pub zip_path: Arc, +} + +pub struct ZipArchiveWrapper { + archive: ZipArchive, +} +// impl ZipArchiveWrapper { +// pub fn clone_with_big_buffer(&self) -> ZipArchive { +// let archive = self.archive.clone(); + +// archive.to_owned(); + +// archive +// } +// } + pub struct HashToPath { - hash2path: HashMap, Arc>, + zip2ziparchive: HashMap, + hash2path: HashMap, BlockLocation>, } impl HashToPath { pub fn new() -> Self { let hash2path = HashMap::new(); - - Self { hash2path } + let zip2ziparchive = HashMap::new(); + Self { + hash2path, + zip2ziparchive, + } } } @@ -63,17 +91,17 @@ pub struct DB { } impl DB { - pub fn new(file: &str, manifest: &str) -> Result { + pub fn new(manifest_bytes: &[u8]) -> Result { // let conn = UnQLite::create(file); // conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; - let manifest: Manifest = serde_json::from_str(manifest)?; + let manifest: Manifest = serde_json::from_slice(manifest_bytes)?; let inner = Arc::new(Mutex::new(HashToPath::new())); let db = DB { inner, manifest }; Ok(db) } - pub fn create_block_id_to_filenames(self, paths: &[String]) -> Result { + pub fn create_block_id_to_filenames(&self, paths: &[PathBuf]) -> Result<()> { // Iterate through dblocks, adding them to the db let pb = ProgressBar::new(paths.len() as u64); pb.set_style( @@ -83,39 +111,64 @@ impl DB { ) .progress_chars("##-"), ); - //let inner = &self.inner; - for zippath in paths { - // In this stage, open the file - - let zipbuf = MyCloneFileReader::new(Path::new(&zippath).to_path_buf())?; - let zip = zip::ZipArchive::new(zipbuf).unwrap(); - - let zip_len = zip.len(); + for zip_path in paths { + self.import_from_zip(zip_path)?; + } - //for zipfile in zip.file_names() + Ok(()) + } - // Convert to a list of paths + pub fn import_from_zip(&self, zip_path: &PathBuf) -> Result<()> { + // In this stage, open the file + let zip_path = Path::new(&zip_path).to_path_buf(); + let config = Arc::new(MyCloneFileConfig { + path: zip_path.clone(), + buf_capacity: AtomicU32::new(1024), + }); + let zipbuf = MyCloneFileReader::new(config.clone())?; + let zip = zip::ZipArchive::new(zipbuf)?; + + let zip_len = zip.len(); + + let arc_zippath = Arc::new(ZipLocation { + path: zip_path.clone(), + path_str: zip_path.to_string_lossy().to_string(), + }); + // Convert to a list of paths + + for (index, file_name) in zip.file_names_ordered().enumerate() { + //let file_in_zip = zip.by_index_raw(file_index)?; + //let file_name = file_in_zip.name().to_string(); + let hash_path = file_name; + let hash = base64::decode_config(&hash_path, base64::URL_SAFE)?; + { + let mut inner = self.inner.lock().unwrap(); + inner.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_zippath.clone(), + file_index: index, + }, + ); + } + } + if false { let mut hvec = Vec::new(); - let (sender, receiver) = crossbeam_channel::unbounded(); + let (sender, receiver) = crossbeam_channel::bounded(zip_len + 1); // make workers - for t in 0..4 { - println!("Make worker {}", t); + for _t in 0..16 { + //println!("Make worker {}", t); let receiver = receiver.clone(); // clone for this thread - let azippath = Arc::new(ZipFilePath { - path: zippath.clone(), - }); - let zippath = zippath.clone(); + //let zip_path = zip_path.clone(); let inner = self.inner.clone(); let mut zip = zip.clone(); + let arc_zippath = arc_zippath.clone(); let handler = std::thread::spawn(move || { - //let mut rng = rand::thread_rng(); // each thread have one - // let zipfile = File::open(&Path::new(&zippath)).unwrap(); - // let zipbuf = BufReader::with_capacity(1 * 1024, zipfile); - // let mut zip = zip::ZipArchive::new(zipbuf).unwrap(); + let mut progress_burst = 0; loop { let r = receiver.recv(); @@ -131,12 +184,29 @@ impl DB { base64::decode_config(&hash_path, base64::URL_SAFE).unwrap(); { let mut inner = inner.lock().unwrap(); - inner.hash2path.insert(hash.into(), azippath.clone()); + inner.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_zippath.clone(), + file_index: file_index, + }, + ); + } + + progress_burst += 1; + if progress_burst > 1000 { + //pb.inc(progress_burst); + println!( + "found hash {}/{} in {:?}", + file_index, zip_len, arc_zippath.path + ); + progress_burst = 0; } } _ => break, } } + //pb.inc(progress_burst); }); hvec.push(handler); @@ -146,38 +216,56 @@ impl DB { sender.send(i).unwrap(); } drop(sender); - // let paths: Vec = (0..zip_len) - // .into_par_iter() - // .map_init( - // || { - // zip - // }, - // |zip, i| , - // ) - // .collect(); + for h in hvec { + h.join().unwrap(); + } + } + { + config + .buf_capacity + .store(32 * 1024, std::sync::atomic::Ordering::Relaxed); + let mut inner = self.inner.lock().unwrap(); + let wrapper = ZipArchiveWrapper { archive: zip }; + let path_str = arc_zippath.path_str.clone(); + inner.zip2ziparchive.insert(path_str, wrapper); + } + // let paths: Vec = (0..zip_len) + // .into_par_iter() + // .map_init( + // || { - // let bytes = zippath.as_bytes(); - // for p in paths { + // zip + // }, + // |zip, i| , + // ) + // .collect(); - // //conn.kv_store(hash, bytes).unwrap(); + // let bytes = zippath.as_bytes(); + // for p in paths { - // //println!("len: {}", hash.len()); - // //println!("zippath:{} hash:{}", zippath, p); + // //conn.kv_store(hash, bytes).unwrap(); - // } - //conn.commit().unwrap(); - for h in hvec { - h.join().unwrap(); - } + // //println!("len: {}", hash.len()); + // //println!("zippath:{} hash:{}", zippath, p); - pb.inc(1); - } + // } + //conn.commit().unwrap(); + Ok(()) + } - Ok(self) + pub fn get_block_id_location(&self, block_id: &str) -> Option { + let key = base64::decode_config(block_id, base64::STANDARD).unwrap(); + let key = SmallVec::from(key); + self.inner + .lock() + .unwrap() + .hash2path + .get(&key) + .map(|v| v.clone()) } - pub fn get_filename_from_block_id(&self, block_id: &str) -> Option { + pub fn get_filename_from_block_id(&self, block_id: &str) -> Option { //let conn = &self.conn; // println!("{}", block_id); // let converted_block_id = base64_url_to_plain(block_id); @@ -189,7 +277,7 @@ impl DB { .unwrap() .hash2path .get(&key) - .map(|v| v.path.clone()) + .map(|v| v.zip_path.path.clone()) // let result = conn.kv_fetch(key); // if let Ok(path_bytes) = result { // Some(String::from_utf8(path_bytes).unwrap()) @@ -198,21 +286,33 @@ impl DB { // } } - pub fn get_content_block(&self, block_id: &str) -> Option> { + pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { + let inner = self.inner.lock().unwrap(); + let zip = inner.zip2ziparchive.get(zip_filename); + + zip.map(|zip| zip.archive.clone()) + } + + pub fn get_content_block(&self, block_id: &str) -> Result>> { let mut output = Vec::new(); - if let Some(filename) = self.get_filename_from_block_id(block_id) { - let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); + + //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); + + let zname = self.get_filename_from_block_id(block_id); + let zname = zname.map(|n| n.to_string_lossy().to_string()); + let zip = zname.map(|zname| self.get_zip_archive(&zname)).flatten(); + if let Some(mut zip) = zip { let mut block = zip .by_name(&base64::encode_config( - &base64::decode(block_id).unwrap(), + &base64::decode(block_id).expect("wrong base64 block_id"), base64::URL_SAFE, )) .unwrap(); - block.read_to_end(&mut output).unwrap(); + block.read_to_end(&mut output)?; - Some(output) + Ok(Some(output)) } else { - None + Ok(None) } } @@ -230,25 +330,34 @@ impl DB { 32 } } - -struct MyCloneFileReader { - path: PathBuf, +pub struct MyCloneFileConfig { + pub path: PathBuf, + /// Changes after the files are indexed. + /// Bigger buf helps with large file reads. + /// Smaller buf does less redundant byte reads from disk when indexing. + pub buf_capacity: AtomicU32, +} +pub struct MyCloneFileReader { + pub config: Arc, buf_reader: BufReader, } impl Clone for MyCloneFileReader { fn clone(&self) -> Self { - Self::new(self.path.clone()).unwrap() + Self::new(self.config.clone()).unwrap() } } impl MyCloneFileReader { - pub fn new(path: PathBuf) -> Result { - let target_file = File::open(&path)?; - let filebuf = BufReader::with_capacity(2 * 1024, target_file); + pub fn new(config: Arc) -> Result { + let target_file = File::open(&config.path)?; + let cap = config + .buf_capacity + .load(std::sync::atomic::Ordering::Relaxed); + let filebuf = BufReader::with_capacity(cap as usize, target_file); Ok(Self { - path, + config: config.clone(), buf_reader: filebuf, }) } @@ -259,10 +368,6 @@ impl Read for MyCloneFileReader { self.buf_reader.read(buf) } - // fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> std::io::Result<()> { - // self.buf_reader.read_buf(buf) - // } - fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { self.buf_reader.read_exact(buf) } @@ -271,10 +376,6 @@ impl Read for MyCloneFileReader { self.buf_reader.read_vectored(bufs) } - // fn is_read_vectored(&self) -> bool { - // self.buf_reader.is_read_vectored() - // } - fn read_to_end(&mut self, buf: &mut Vec) -> std::io::Result { self.buf_reader.read_to_end(buf) } @@ -293,3 +394,13 @@ impl Seek for MyCloneFileReader { self.buf_reader.stream_position() } } + +impl BufRead for MyCloneFileReader { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + self.buf_reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.buf_reader.consume(amt) + } +} diff --git a/src/main.rs b/src/main.rs index 48eb2d1..d86c4d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,6 @@ mod database; mod stripbom; use blockid::*; -use chrono::Duration; use clap::Parser; use database::*; use eyre::eyre; @@ -14,7 +13,7 @@ use rayon::prelude::*; use std::fs; use std::fs::File; use std::io::Read; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use zip; @@ -33,6 +32,9 @@ struct CliArgs { #[arg(short, long)] cpu_count: Option, + + #[arg(short, long)] + progress_bar: Option, } fn main() { @@ -53,13 +55,64 @@ fn main() { } } +fn filename_ends_with>(path: P, suffix: &str) -> bool { + path.as_ref() + .file_name() + .map(|name| name.to_str()) + .flatten() + .map(|name| name.ends_with(suffix)) + .unwrap_or(false) +} + +fn path_is_dlist_zip>(path: P) -> bool { + filename_ends_with(path, "dlist.zip") +} +fn path_is_dblock_zip>(path: P) -> bool { + filename_ends_with(path, "dblock.zip") +} + +/// Open dlist file and parse json inside +fn parse_dlist_file>(dlist_path: P) -> Result> { + let dlist_reader = File::open(dlist_path.as_ref()) + .wrap_err_with(|| format!("open {:?}", dlist_path.as_ref()))?; + let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; + let filelist_name = "filelist.json"; + let mut dlist_file = dlist_zip.by_name(filelist_name)?; + let mut dlist_contents = Vec::new(); + dlist_file.read_to_end(&mut dlist_contents)?; + + let list = parse_dlist(&dlist_contents).wrap_err_with(|| { + format!( + "parse_dlist {:?} / {:?}", + dlist_path.as_ref(), + filelist_name + ) + })?; + + Ok(list) +} + +/// Open Manifest from zip +fn read_manifest>(dlist_path: P) -> Result> { + let manifest_file = File::open(dlist_path.as_ref())?; + let mut manifest_zip = zip::ZipArchive::new(manifest_file)?; + let mut manifest_file = manifest_zip.by_name("manifest")?; + let mut manifest_contents = String::new(); + manifest_file + .read_to_string(&mut manifest_contents) + .wrap_err_with(|| format!("read manifest from {:?}", dlist_path.as_ref()))?; + let manifest_contents = manifest_contents.strip_bom(); + let manifest_contents = manifest_contents.trim(); + Ok(manifest_contents.into()) +} + fn run() -> Result<()> { let args = CliArgs::parse(); let backup_dir = args.backup_dir.trim(); let restore_dir = args.restore_dir.trim(); - let db_location = Path::join(Path::new(restore_dir), Path::new("index.db")); - let db_location = db_location.to_str().unwrap(); + // let db_location = Path::join(Path::new(restore_dir), Path::new("index.db")); + // let db_location = db_location.to_str().unwrap(); let cpu_count: usize = args.cpu_count.unwrap_or_else(|| num_cpus::get()); println!(); @@ -71,47 +124,28 @@ fn run() -> Result<()> { .unwrap(); // Find newest dlist - let mut dlist_file_names: Vec = fs::read_dir(&backup_dir) - .unwrap() + let mut dlist_file_paths: Vec = fs::read_dir(&backup_dir)? .filter_map(Result::ok) - .filter(|f| f.path().to_str().unwrap().ends_with("dlist.zip")) - .map(|f| f.path().to_str().unwrap().to_string()) + .filter(|f| path_is_dlist_zip(f.path())) + //.map(|f| (f.metadata().map(|m| m.modified()), f)) + .map(|f| f.path().to_path_buf()) .collect(); - dlist_file_names.sort(); + dlist_file_paths.sort(); - let dlist = dlist_file_names + let newest_dlist = dlist_file_paths .last() .ok_or_else(|| eyre!("last modified dlist file not found"))?; - println!("{} appears to be newest dlist, using it.", dlist); + println!( + "Newest: {:?} appears to be newest dlist, using it.", + newest_dlist + ); println!("Parsing dlist"); - - // Open dlist file - let file_entries = { - let dlist_reader = File::open(dlist.clone()).wrap_err_with(|| format!("open {}", dlist))?; - let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; - let mut dlist_file = dlist_zip.by_name("filelist.json")?; - let mut dlist_contents = Vec::new(); - dlist_file.read_to_end(&mut dlist_contents)?; - - parse_dlist(&dlist_contents).wrap_err_with(|| format!("parse_dlist {}", dlist))? - }; + let mut file_entries = parse_dlist_file(newest_dlist)?; println!("Parsing manifest"); - // Open Manifest - let manifest_contents = { - let manifest_file = File::open(dlist.clone())?; - let mut manifest_zip = zip::ZipArchive::new(manifest_file)?; - let mut manifest_file = manifest_zip.by_name("manifest")?; - let mut manifest_contents = String::new(); - manifest_file - .read_to_string(&mut manifest_contents) - .wrap_err("read manifest")?; - let manifest_contents = manifest_contents.strip_bom(); - let manifest_contents = manifest_contents.trim(); - manifest_contents.to_owned() - }; + let manifest_contents = read_manifest(newest_dlist)?; let file_count = file_entries.iter().filter(|f| f.is_file()).count(); println!("{} files to be restored", file_count); @@ -120,11 +154,11 @@ fn run() -> Result<()> { println!(); // Get list of dblocks - let zip_file_names: Vec = fs::read_dir(backup_dir) + let zip_file_names: Vec = fs::read_dir(backup_dir) .unwrap() .filter_map(Result::ok) - .map(|f| f.path().to_str().unwrap().to_string()) - .filter(|f| f.ends_with("dblock.zip")) + .filter(|f| path_is_dblock_zip(f.path())) + .map(|f| f.path().to_path_buf()) .collect(); println!("Found {} dblocks", zip_file_names.len()); @@ -132,30 +166,60 @@ fn run() -> Result<()> { // Open dblock db connection and build db println!(); println!("Indexing dblocks"); - let dblock_db = - DB::new(db_location, &manifest_contents)?.create_block_id_to_filenames(&zip_file_names)?; + let dblock_db = DB::new(&manifest_contents)?; + dblock_db.create_block_id_to_filenames(&zip_file_names)?; + let show_progress = args.progress_bar.unwrap_or_default(); println!("Restoring directory structure"); - let mut pb = ProgressBar::new(folder_count as u64); + let mut pb = if show_progress { + Some(ProgressBar::new(folder_count as u64)) + } else { + None + }; + for d in file_entries.iter().filter(|f| f.is_folder()) { - d.restore_file(&dblock_db, &restore_dir); - pb.inc(); + d.restore_file(&dblock_db, &restore_dir) + .wrap_err("restoring dir")?; + if let Some(pb) = &mut pb { + pb.inc(); + } } println!(); + println!("Sorting file_entries"); + + sort_files_sequentially(&mut file_entries, &dblock_db); + println!("Restoring files"); - let pb = Arc::new(Mutex::new(ProgressBar::new(file_count as u64))); + let pb = if show_progress { + Some(Arc::new(Mutex::new(ProgressBar::new(file_count as u64)))) + } else { + None + }; file_entries - .par_iter() + //.par_iter() + .iter() .filter(|f| f.is_file()) - .for_each(|f| { - f.restore_file(&dblock_db, &restore_dir); - pb.lock().unwrap().inc(); - }); + .par_bridge() + //.iter() + .try_for_each(|f| -> Result<()> { + f.restore_file(&dblock_db, &restore_dir) + .wrap_err("restoring file entry")?; + if let Some(pb) = &pb { + pb.lock().unwrap().inc(); + } + Ok(()) + })?; Ok(()) } +/// Not necessary, but useful to speed up file reads from HDD +/// from like 200 Mbit/s to 700 Mbit/s +fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DB) { + file_entries.sort_by(|a, b| a.compare(b, dblock_db)); +} + #[cfg(feature = "dhat-heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; From 719a29cda38614058805302b218c791cf261c204 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Wed, 11 Jan 2023 16:51:02 +0100 Subject: [PATCH 04/38] main hashmap is now optional --- .gitignore | 3 +- src/blockid.rs | 25 ++++-- src/database.rs | 205 ++++++++++++++++++++++++++++++------------------ src/main.rs | 23 +++--- src/stripbom.rs | 43 +++++++++- 5 files changed, 203 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index 0821766..6dd50ef 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ /debug_*.json /testout /dhat-heap.json -.vscode/settings.json \ No newline at end of file +.vscode/settings.json +testrun.sh \ No newline at end of file diff --git a/src/blockid.rs b/src/blockid.rs index 3b44cf3..a95276e 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -1,12 +1,12 @@ use crate::database::BlockLocation; use crate::database::DB; -use crate::stripbom::StripBomBytes; +use crate::stripbom::strip_bom_from_bufread; use base64; use eyre::Context; use eyre::Result; use serde::Deserialize; use serde_json; -use serde_json::de::SliceRead; +use serde_json::de::IoRead; use serde_json::Deserializer; use eyre::eyre; @@ -129,7 +129,7 @@ impl FileEntry { let a = self.get_first_bytes_location(db); let b = othr.get_first_bytes_location(db); - a.cmp(&b) + a.cmp(&b).then_with(|| self.cmp(othr)) } pub fn restore_file(&self, db: &DB, restore_path: &str) -> Result<()> { @@ -150,7 +150,7 @@ impl FileEntry { if self.block_lists.is_empty() { let loc = db.get_block_id_location(hash); println!( - "restoring file {:?}, index:{:?}", + "restoring file (single) {:?}, index:{:?}", relative_file_path, loc.map(|loc| loc.file_index) ); @@ -175,7 +175,7 @@ impl FileEntry { .map(|hash| db.get_block_id_location(hash)) .flatten(); println!( - "restoring file {:?}, index:{:?}", + "restoring file (blocks) {:?}, index:{:?}", relative_file_path, loc.map(|loc| loc.file_index) ); @@ -236,13 +236,24 @@ pub(self) struct IEntry { pub(self) blocklists: Option>, } +#[allow(unused)] /// Accepts the dlist as a string (must be read in first) /// Returns a Vec of FileEntrys pub fn parse_dlist(dlist: &[u8]) -> Result> { + let file_entries = parse_dlist_read(dlist)?; + + Ok(file_entries) +} + +/// Accepts the dlist as a Read trait +/// Returns a Vec of FileEntrys +pub fn parse_dlist_read<'a, R: BufRead>(mut rdr: R) -> Result> { let mut file_entries = Vec::new(); - let read = SliceRead::new(dlist.strip_bom()); - let mut de = Deserializer::new(read); + strip_bom_from_bufread(&mut rdr)?; + + let iread = IoRead::new(rdr); + let mut de = Deserializer::new(iread); let entry_list: Vec = serde_path_to_error::deserialize(&mut de).wrap_err("deserialize entry_list")?; diff --git a/src/database.rs b/src/database.rs index cb1dd9b..5fe5bbb 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,6 +1,9 @@ use base64; +use eyre::Context; use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; +use rayon::prelude::IntoParallelRefIterator; +use rayon::prelude::ParallelIterator; use serde::Deserialize; use serde_json; use smallvec::SmallVec; @@ -49,7 +52,7 @@ pub struct ZipLocation { #[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] pub struct BlockLocation { /// Which file inside the zip - pub file_index: usize, + pub file_index: u32, /// Which dblock.zip file pub zip_path: Arc, @@ -58,45 +61,105 @@ pub struct BlockLocation { pub struct ZipArchiveWrapper { archive: ZipArchive, } -// impl ZipArchiveWrapper { -// pub fn clone_with_big_buffer(&self) -> ZipArchive { -// let archive = self.archive.clone(); - -// archive.to_owned(); - -// archive -// } -// } pub struct HashToPath { - zip2ziparchive: HashMap, + /// Maps hash (without base64) to location in dblock.zip + /// + /// May be faster, but it's memory-intensive hash2path: HashMap, BlockLocation>, } - impl HashToPath { pub fn new() -> Self { - let hash2path = HashMap::new(); + Self { + hash2path: HashMap::new(), + } + } + + pub fn get_zip_path_by_base64(&self, block_id: &str) -> Option { + let key = base64::decode_config(block_id, base64::STANDARD).ok()?; + let key = SmallVec::from(key); + + self.hash2path.get(&key).map(|v| v.zip_path.path.clone()) + } + + pub fn get_location_by_base64(&self, block_id: &str) -> Option { + let key = base64::decode_config(block_id, base64::STANDARD).ok()?; + let key = SmallVec::from(key); + self.hash2path.get(&key).map(|v| v.clone()) + } +} +pub struct HashToBlocks { + /// Maps zip file name to a singleton zip reader + zip2ziparchive: HashMap, + + hash2path: Option, +} + +impl HashToBlocks { + pub fn new(use_hash_to_path: bool) -> Self { + let hash2path = if use_hash_to_path { + Some(HashToPath::new()) + } else { + None + }; let zip2ziparchive = HashMap::new(); Self { hash2path, zip2ziparchive, } } + + // pub fn get_zip_path_by_base64(&self, block_id: &str) -> Option { + // if let Some(hash2path) = &self.hash2path { + // hash2path.get_zip_path_by_base64(block_id) + // } else { + // None + // } + // } + + pub fn get_location_by_base64(&self, block_id: &str) -> Option { + if let Some(hash2path) = &self.hash2path { + hash2path.get_location_by_base64(block_id) + } else { + None + } + } + + pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { + let zip = self.zip2ziparchive.get(zip_filename); + + zip.map(|zip| zip.archive.clone()) + } + + pub fn get_zip_by_block_id(&self, block_id: &str) -> Option> { + if let Some(hash2path) = &self.hash2path { + let zname = hash2path.get_zip_path_by_base64(block_id); + let zname = zname.map(|n| n.to_string_lossy().to_string()); + let zipa = zname.map(|zname| self.get_zip_archive(&zname)).flatten(); + zipa + } else { + for ziparch in self.zip2ziparchive.values() { + if ziparch.archive.contains_file_name(block_id) { + return Some(ziparch.archive.clone()); + } + } + None + } + } } pub struct DB { - // conn: UnQLite, - inner: Arc>, + inner: Arc>, manifest: Manifest, } impl DB { - pub fn new(manifest_bytes: &[u8]) -> Result { + pub fn new(manifest_bytes: &[u8], use_hash_to_path: bool) -> Result { // let conn = UnQLite::create(file); // conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; let manifest: Manifest = serde_json::from_slice(manifest_bytes)?; - let inner = Arc::new(Mutex::new(HashToPath::new())); + let inner = Arc::new(Mutex::new(HashToBlocks::new(use_hash_to_path))); let db = DB { inner, manifest }; Ok(db) } @@ -111,9 +174,9 @@ impl DB { ) .progress_chars("##-"), ); - for zip_path in paths { - self.import_from_zip(zip_path)?; - } + paths + .par_iter() + .try_for_each(|zip_path| self.import_from_zip(zip_path))?; Ok(()) } @@ -142,14 +205,19 @@ impl DB { let hash_path = file_name; let hash = base64::decode_config(&hash_path, base64::URL_SAFE)?; { + if hash.len() > 32 { + println!("warn: hash len:{} requires heap alloc", hash.len()); + } let mut inner = self.inner.lock().unwrap(); - inner.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_zippath.clone(), - file_index: index, - }, - ); + if let Some(hash2path) = &mut inner.hash2path { + hash2path.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_zippath.clone(), + file_index: index as u32, + }, + ); + } } } @@ -184,13 +252,16 @@ impl DB { base64::decode_config(&hash_path, base64::URL_SAFE).unwrap(); { let mut inner = inner.lock().unwrap(); - inner.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_zippath.clone(), - file_index: file_index, - }, - ); + + if let Some(hash2path) = &mut inner.hash2path { + hash2path.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_zippath.clone(), + file_index: file_index as u32, + }, + ); + } } progress_burst += 1; @@ -255,60 +326,40 @@ impl DB { } pub fn get_block_id_location(&self, block_id: &str) -> Option { - let key = base64::decode_config(block_id, base64::STANDARD).unwrap(); - let key = SmallVec::from(key); - self.inner - .lock() - .unwrap() - .hash2path - .get(&key) - .map(|v| v.clone()) + self.inner.lock().unwrap().get_location_by_base64(block_id) } - pub fn get_filename_from_block_id(&self, block_id: &str) -> Option { - //let conn = &self.conn; - // println!("{}", block_id); - // let converted_block_id = base64_url_to_plain(block_id); - let key = base64::decode_config(block_id, base64::STANDARD).unwrap(); - let key = SmallVec::from(key); + // pub fn get_zip_path_from_block_id(&self, block_id: &str) -> Option { + // self.inner.lock().unwrap().get_zip_path_by_base64(block_id) + // } - self.inner - .lock() - .unwrap() - .hash2path - .get(&key) - .map(|v| v.zip_path.path.clone()) - // let result = conn.kv_fetch(key); - // if let Ok(path_bytes) = result { - // Some(String::from_utf8(path_bytes).unwrap()) - // } else { - // None - // } - } + // pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { + // self.inner.lock().unwrap().get_zip_archive(zip_filename) - pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { - let inner = self.inner.lock().unwrap(); - let zip = inner.zip2ziparchive.get(zip_filename); + // } - zip.map(|zip| zip.archive.clone()) + pub fn get_zip_by_block_id(&self, block_id: &str) -> Option> { + self.inner.lock().unwrap().get_zip_by_block_id(block_id) } pub fn get_content_block(&self, block_id: &str) -> Result>> { let mut output = Vec::new(); - //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); + let name_reencoded: String = base64::encode_config( + &base64::decode(block_id).expect("wrong base64 block_id"), + base64::URL_SAFE, + ); - let zname = self.get_filename_from_block_id(block_id); - let zname = zname.map(|n| n.to_string_lossy().to_string()); - let zip = zname.map(|zname| self.get_zip_archive(&zname)).flatten(); - if let Some(mut zip) = zip { - let mut block = zip - .by_name(&base64::encode_config( - &base64::decode(block_id).expect("wrong base64 block_id"), - base64::URL_SAFE, - )) - .unwrap(); - block.read_to_end(&mut output)?; + //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); + let ziparch = self.get_zip_by_block_id(&name_reencoded); + + if let Some(mut ziparch) = ziparch { + let mut block = ziparch + .by_name(&name_reencoded) + .wrap_err("block file by name not found even though we indexed it before")?; + block + .read_to_end(&mut output) + .wrap_err_with(|| format!("reading block file {:?}", block_id))?; Ok(Some(output)) } else { diff --git a/src/main.rs b/src/main.rs index d86c4d9..82c10e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use pbr::ProgressBar; use rayon::prelude::*; use std::fs; use std::fs::File; -use std::io::Read; +use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use zip; @@ -35,6 +35,10 @@ struct CliArgs { #[arg(short, long)] progress_bar: Option, + + /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. + #[arg(short, long)] + hash_to_path: bool, } fn main() { @@ -42,7 +46,7 @@ fn main() { std::thread::spawn(|| { let _profiler = dhat::Profiler::new_heap(); - std::thread::sleep(std::time::Duration::from_secs(60)); + std::thread::sleep(std::time::Duration::from_secs(4 * 60)); }); std::thread::sleep(std::time::Duration::from_millis(100)); @@ -77,11 +81,11 @@ fn parse_dlist_file>(dlist_path: P) -> Result> { .wrap_err_with(|| format!("open {:?}", dlist_path.as_ref()))?; let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; let filelist_name = "filelist.json"; - let mut dlist_file = dlist_zip.by_name(filelist_name)?; - let mut dlist_contents = Vec::new(); - dlist_file.read_to_end(&mut dlist_contents)?; - - let list = parse_dlist(&dlist_contents).wrap_err_with(|| { + let dlist_file = dlist_zip.by_name(filelist_name)?; + // let mut dlist_contents = Vec::new(); + // dlist_file.read_to_end(&mut dlist_contents)?; + let bufrdr = BufReader::with_capacity(32 * 1024, dlist_file); + let list = parse_dlist_read(bufrdr).wrap_err_with(|| { format!( "parse_dlist {:?} / {:?}", dlist_path.as_ref(), @@ -115,11 +119,10 @@ fn run() -> Result<()> { // let db_location = db_location.to_str().unwrap(); let cpu_count: usize = args.cpu_count.unwrap_or_else(|| num_cpus::get()); - println!(); // Set CPU count rayon::ThreadPoolBuilder::new() - .num_threads(cpu_count) + .num_threads(cpu_count.min(4)) .build_global() .unwrap(); @@ -166,7 +169,7 @@ fn run() -> Result<()> { // Open dblock db connection and build db println!(); println!("Indexing dblocks"); - let dblock_db = DB::new(&manifest_contents)?; + let dblock_db = DB::new(&manifest_contents, args.hash_to_path)?; dblock_db.create_block_id_to_filenames(&zip_file_names)?; let show_progress = args.progress_bar.unwrap_or_default(); diff --git a/src/stripbom.rs b/src/stripbom.rs index 46f57a6..73dd3ce 100644 --- a/src/stripbom.rs +++ b/src/stripbom.rs @@ -1,3 +1,6 @@ +use eyre::Result; +use std::io::{BufRead, Read}; + pub trait StripBom { fn strip_bom(&self) -> &str; } @@ -18,16 +21,54 @@ impl StripBom for String { } } +fn starts_with_bom_bytes(s: &[u8]) -> bool { + s.starts_with(&[0xEF, 0xBB, 0xBF]) +} + pub trait StripBomBytes { fn strip_bom(&self) -> &[u8]; } impl StripBomBytes for [u8] { fn strip_bom<'a>(&'a self) -> &'a [u8] { - if self.starts_with(&[0xEF, 0xBB, 0xBF]) { + if starts_with_bom_bytes(self) { &self[3..] } else { &self[..] } } } + +pub fn strip_bom_from_bufread(mut inner: R) -> Result<()> { + let buf = inner.fill_buf()?; + if buf.len() >= 3 && starts_with_bom_bytes(buf) { + //println!("removing bom"); + inner.consume(3); + } + + // here we ignore case of 1 or 2 bytes of BOM + Ok(()) +} + +pub struct StripBomReader { + pub first_bytes: bool, + pub inner: R, +} + +impl StripBomReader { + #[allow(unused)] + pub fn new(mut inner: R) -> Result> { + strip_bom_from_bufread(&mut inner)?; + + Ok(Self { + first_bytes: true, + inner, + }) + } +} + +impl Read for StripBomReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.inner.read(buf) + } +} From b4bf5c4ea47cd62e0ae7eb742029e82ec638ece4 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:51:57 +0100 Subject: [PATCH 05/38] refactor --- .gitignore | 2 +- Cargo.lock | 454 +++++++++++++++++++++++++++------------------- src/blockhash.rs | 66 +++++++ src/blockid.rs | 156 +++------------- src/database.rs | 84 +++++---- src/hexdisplay.rs | 60 ++++++ src/main.rs | 30 +-- src/restoring.rs | 102 +++++++++++ src/sorting.rs | 30 +++ 9 files changed, 622 insertions(+), 362 deletions(-) create mode 100644 src/blockhash.rs create mode 100644 src/hexdisplay.rs create mode 100644 src/restoring.rs create mode 100644 src/sorting.rs diff --git a/.gitignore b/.gitignore index 6dd50ef..b9c6d36 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,6 @@ /debug_*.json /testout -/dhat-heap.json +/dhat-heap*.json .vscode/settings.json testrun.sh \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 1bb9d45..9129060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,32 +39,32 @@ dependencies = [ ] [[package]] -name = "ansi_term" -version = "0.11.0" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "winapi 0.3.7", + "libc", ] [[package]] -name = "arrayvec" -version = "0.4.10" +name = "ansi_term" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "nodrop", + "winapi", ] [[package]] name = "atty" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ + "hermit-abi 0.1.19", "libc", - "termion", - "winapi 0.3.7", + "winapi", ] [[package]] @@ -111,9 +111,9 @@ checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443" dependencies = [ "bitflags", "cexpr", - "cfg-if 0.1.7", + "cfg-if 0.1.10", "clang-sys", - "clap 2.33.0", + "clap 2.34.0", "env_logger", "lazy_static", "lazycell", @@ -142,6 +142,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + [[package]] name = "byteorder" version = "1.4.3" @@ -189,9 +195,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "cfg-if" @@ -201,13 +207,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.6" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ + "iana-time-zone", + "js-sys", "num-integer", "num-traits", - "time 0.1.42", + "time 0.1.45", + "wasm-bindgen", + "winapi", ] [[package]] @@ -232,9 +242,9 @@ dependencies = [ [[package]] name = "clap" -version = "2.33.0" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", @@ -283,33 +293,26 @@ dependencies = [ ] [[package]] -name = "clicolors-control" -version = "1.0.0" +name = "codespan-reporting" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73abfd4c73d003a674ce5d2933fca6ce6c42480ea84a5ffe0a2dc39ed56300f9" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ - "atty", - "lazy_static", - "libc", - "winapi 0.3.7", + "termcolor", + "unicode-width", ] [[package]] name = "console" -version = "0.7.5" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf3720d3f3fc30b721ef1ae54e13af3264af4af39dc476a8de56a6ee1e2184b" +checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" dependencies = [ - "atty", - "clicolors-control", "encode_unicode", "lazy_static", "libc", - "parking_lot", - "regex", - "termios", "unicode-width", - "winapi 0.3.7", + "windows-sys", ] [[package]] @@ -318,6 +321,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.5" @@ -343,41 +352,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.14", + "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.2.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ + "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.2.2", + "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.3.1" +version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927121f5407de9956180ff5e936fe3cf4324279280001cd56b669d28ee7e9150" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ - "arrayvec", - "cfg-if 0.1.7", - "crossbeam-utils 0.2.2", - "lazy_static", + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", "memoffset", - "nodrop", - "scopeguard 0.3.3", -] - -[[package]] -name = "crossbeam-utils" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" -dependencies = [ - "cfg-if 0.1.7", + "scopeguard", ] [[package]] @@ -399,6 +398,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "cxx" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dhat" version = "0.3.2" @@ -428,15 +471,15 @@ dependencies = [ [[package]] name = "either" -version = "1.5.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "encode_unicode" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "env_logger" @@ -459,7 +502,7 @@ checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" dependencies = [ "errno-dragonfly", "libc", - "winapi 0.3.7", + "winapi", ] [[package]] @@ -520,6 +563,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -547,6 +599,30 @@ dependencies = [ "quick-error", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "indenter" version = "0.3.3" @@ -582,18 +658,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "io-lifetimes", "rustix", "windows-sys", ] -[[package]] -name = "itoa" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" - [[package]] name = "itoa" version = "1.0.5" @@ -610,13 +680,12 @@ dependencies = [ ] [[package]] -name = "kernel32-sys" -version = "0.2.2" +name = "js-sys" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "wasm-bindgen", ] [[package]] @@ -644,7 +713,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if 1.0.0", - "winapi 0.3.7", + "winapi", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", ] [[package]] @@ -660,7 +738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", - "scopeguard 1.1.0", + "scopeguard", ] [[package]] @@ -680,9 +758,12 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.2.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] [[package]] name = "miniz_oxide" @@ -703,12 +784,6 @@ dependencies = [ "sys-info", ] -[[package]] -name = "nodrop" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" - [[package]] name = "nom" version = "5.1.2" @@ -721,25 +796,30 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.39" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ + "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.6" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] [[package]] name = "num_cpus" -version = "1.10.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ + "hermit-abi 0.2.6", "libc", ] @@ -752,17 +832,11 @@ dependencies = [ "num-traits", ] -[[package]] -name = "numtoa" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" - [[package]] name = "object" -version = "0.30.1" +version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d864c91689fdc196779b98dba0aceac6118594c2df6ee5d943eb6a8df4d107a" +checksum = "2b8c786513eb403643f2a88c244c2aaa270ef2153f55094587d0c48a3cf22a83" dependencies = [ "memchr", ] @@ -797,13 +871,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", "windows-sys", ] @@ -839,15 +913,14 @@ dependencies = [ [[package]] name = "pbr" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb73390ab68d81992bd994d145f697451bb0b54fd39738e72eef32458ad6907" +checksum = "ff5751d87f7c00ae6403eb1fcbba229b9c76c9a30de8c1cf87182177b168cea2" dependencies = [ - "kernel32-sys", + "crossbeam-channel", "libc", - "termion", - "time 0.1.42", - "winapi 0.2.8", + "time 0.1.45", + "winapi", ] [[package]] @@ -918,33 +991,26 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "rayon" -version = "1.0.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373814f27745b2686b350dd261bfd24576a6fb0e2c5919b3a2b6005f820b0473" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.4.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" +checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" dependencies = [ + "crossbeam-channel", "crossbeam-deque", - "lazy_static", - "libc", + "crossbeam-utils", "num_cpus", ] -[[package]] -name = "redox_syscall" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -954,15 +1020,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_termios" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -dependencies = [ - "redox_syscall 0.1.54", -] - [[package]] name = "regex" version = "1.7.1" @@ -1030,21 +1087,21 @@ dependencies = [ [[package]] name = "ryu" -version = "0.2.7" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "scopeguard" -version = "0.3.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "scopeguard" -version = "1.1.0" +name = "scratch" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "serde" @@ -1068,11 +1125,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.39" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ - "itoa 0.4.3", + "itoa", "ryu", "serde", ] @@ -1168,27 +1225,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "termion" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" -dependencies = [ - "libc", - "numtoa", - "redox_syscall 0.1.54", - "redox_termios", -] - -[[package]] -name = "termios" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" -dependencies = [ - "libc", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -1206,13 +1242,13 @@ checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" [[package]] name = "time" -version = "0.1.42" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", - "redox_syscall 0.1.54", - "winapi 0.3.7", + "wasi", + "winapi", ] [[package]] @@ -1221,7 +1257,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", "time-macros", @@ -1256,9 +1292,9 @@ checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-width" -version = "0.1.5" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unqlite" @@ -1274,9 +1310,9 @@ dependencies = [ [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -1284,6 +1320,66 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + [[package]] name = "which" version = "3.1.1" @@ -1295,26 +1391,14 @@ dependencies = [ [[package]] name = "winapi" -version = "0.2.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - -[[package]] -name = "winapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -1327,7 +1411,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.7", + "winapi", ] [[package]] @@ -1402,7 +1486,7 @@ dependencies = [ "bzip2", "constant_time_eq", "crc32fast", - "crossbeam-utils 0.8.14", + "crossbeam-utils", "flate2", "hmac", "pbkdf2", diff --git a/src/blockhash.rs b/src/blockhash.rs new file mode 100644 index 0000000..15ac5fd --- /dev/null +++ b/src/blockhash.rs @@ -0,0 +1,66 @@ +use std::{cell::RefCell, fmt::Display}; + +use smallvec::SmallVec; + +use crate::hexdisplay::HexDisplayBytes; +thread_local! { + pub static BASE64_DECODE_BUF: RefCell> = RefCell::new(Vec::with_capacity(64)); + +} + +#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] +pub struct BlockIdHash { + pub hash: SmallVec<[u8; 32]>, +} + +impl Display for BlockIdHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", HexDisplayBytes(self.hash.as_slice())) + } +} + +impl BlockIdHash { + pub fn from_bytes(b: &[u8]) -> Option { + if b.len() != 32 { + return None; + } + Some(BlockIdHash { + hash: SmallVec::from_slice(b), + }) + } + + pub fn from_base64(block_id_str: &str) -> Option { + Self::from_base64_config(block_id_str, base64::STANDARD) + } + #[allow(unused)] + pub fn from_base64_urlsafe(block_id_str: &str) -> Option { + Self::from_base64_config(block_id_str, base64::URL_SAFE) + } + + pub fn from_base64_config(block_id_str: &str, config: base64::Config) -> Option { + BASE64_DECODE_BUF.with(|b| -> Option { + let buffer: &mut Vec = &mut b.borrow_mut(); + assert!(block_id_str.len() < buffer.capacity()); + base64::decode_config_buf(block_id_str, config, buffer).ok()?; + let hash = BlockIdHash { + hash: SmallVec::from_slice(&buffer), + }; + buffer.clear(); + Some(hash) + }) + } + + #[allow(unused)] + pub fn as_base64<'a>(&self, buf: &'a mut [u8]) -> &'a str { + self.as_base64_config(base64::STANDARD, buf) + } + pub fn as_base64_urlsafe<'a>(&self, buf: &'a mut [u8]) -> &'a str { + self.as_base64_config(base64::URL_SAFE, buf) + } + pub fn as_base64_config<'a>(&self, config: base64::Config, buf: &'a mut [u8]) -> &'a str { + let encoded_len = base64::encode_config_slice(self.hash.as_slice(), config, &mut buf[..]); + //debug_assert_eq!(encoded_len, buf.len()); + + std::str::from_utf8(&buf[..encoded_len]).expect("Invalid UTF8") + } +} diff --git a/src/blockid.rs b/src/blockid.rs index a95276e..a4880aa 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -1,7 +1,5 @@ -use crate::database::BlockLocation; -use crate::database::DB; +use crate::blockhash::BlockIdHash; use crate::stripbom::strip_bom_from_bufread; -use base64; use eyre::Context; use eyre::Result; use serde::Deserialize; @@ -10,18 +8,13 @@ use serde_json::de::IoRead; use serde_json::Deserializer; use eyre::eyre; -use std::cmp::Ordering; -use std::fs; -use std::fs::File; + use std::io::prelude::*; -use std::io::SeekFrom; -use std::io::Write; -use std::path::Path; #[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] pub enum FileType { File { - hash: String, + hash: BlockIdHash, size: i64, time: String, }, @@ -57,13 +50,13 @@ impl FileType { #[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct FileEntry { - path: String, + pub path: String, #[allow(unused)] - metahash: String, + pub metahash: String, #[allow(unused)] - metasize: i64, - file_type: FileType, - block_lists: Vec, + pub metasize: i64, + pub file_type: FileType, + pub block_lists: Vec, } impl FileEntry { @@ -71,14 +64,26 @@ impl FileEntry { let path = ientry.path.clone(); let metahash = ientry.metahash.clone(); let metasize = ientry.metasize; - let block_lists = if let Some(blocks) = &ientry.blocklists { - blocks.clone() - } else { - Vec::new() + let mut block_lists = Vec::new(); + + if let Some(blocks) = &ientry.blocklists { + for block in blocks { + block_lists.push( + BlockIdHash::from_base64(&block) + .ok_or_else(|| eyre!("blocklists BlockIdHash::from_base64 fail"))?, + ); + } }; let file_type = match ientry.filetype.as_ref() { "File" => FileType::File { - hash: ientry.hash.clone().ok_or_else(|| eyre!("hash not found"))?, + hash: ientry + .hash + .as_ref() + .map(|hash| { + BlockIdHash::from_base64(&hash) + .ok_or_else(|| eyre!("ientry.hash BlockIdHash::from_base64 fail")) + }) + .ok_or_else(|| eyre!("hash not found"))??, size: ientry.size.clone().ok_or_else(|| eyre!("size not found"))?, time: ientry.time.clone().ok_or_else(|| eyre!("time not found"))?, }, @@ -107,117 +112,6 @@ impl FileEntry { pub fn is_folder(&self) -> bool { self.file_type.is_folder() } - - /// Optional. Used for sorting. - pub fn get_first_bytes_location(&self, db: &DB) -> Option { - match &self.file_type { - FileType::File { hash, .. } => { - if self.block_lists.is_empty() { - db.get_block_id_location(&hash) - } else { - let first = self.block_lists.first(); - - first.map(|bid| db.get_block_id_location(bid)).flatten() - } - } - _ => None, - } - } - - /// Optional. Used for sorting. - pub fn compare(&self, othr: &FileEntry, db: &DB) -> Ordering { - let a = self.get_first_bytes_location(db); - let b = othr.get_first_bytes_location(db); - - a.cmp(&b).then_with(|| self.cmp(othr)) - } - - pub fn restore_file(&self, db: &DB, restore_path: &str) -> Result<()> { - let root_path = Path::new(restore_path); - let dfile_path = &self.path[0..]; - let dfile_path = dfile_path.replacen(":\\", "\\", 1); - let dfile_path = dfile_path.replace("\\", "/"); - let relative_file_path = Path::new(&dfile_path); - - let path = Path::join(root_path, relative_file_path); - - match &self.file_type { - FileType::Folder { .. } => { - fs::create_dir_all(path)?; - } - FileType::File { hash, size, .. } => { - // Small files only have one block - if self.block_lists.is_empty() { - let loc = db.get_block_id_location(hash); - println!( - "restoring file (single) {:?}, index:{:?}", - relative_file_path, - loc.map(|loc| loc.file_index) - ); - - let mut out_file = File::create(path.clone())?; - let block = db.get_content_block(hash)?; - if let Some(block) = block { - out_file - .write_all(block.as_ref()) - .wrap_err("write single-block file")?; - } else if *size > 0 { - println!( - "Missing block {} for {}", - hash, - path.to_str().unwrap_or("not utf8?") - ); - } - } else { - let loc = self - .block_lists - .first() - .map(|hash| db.get_block_id_location(hash)) - .flatten(); - println!( - "restoring file (blocks) {:?}, index:{:?}", - relative_file_path, - loc.map(|loc| loc.file_index) - ); - let mut out_file = File::create(path.clone())?; - // Each blockid points to a list of blockids - for (blhi, blh) in self.block_lists.iter().enumerate() { - let blockhashoffset = blhi * db.offset_size(); - let binary_hashes = db.get_content_block(blh)?; - if let Some(binary_hashes) = binary_hashes { - for (bi, bhash) in binary_hashes.chunks(db.hash_size()).enumerate() { - let bhash = base64::encode(bhash); - let block = db.get_content_block(&bhash)?; - - if let Some(block) = block { - out_file - .seek(SeekFrom::Start( - (blockhashoffset + bi * db.block_size()) as u64, - )) - .wrap_err("seek blockhashoffset + bi * db.block_size()")?; - out_file.write_all(&block).wrap_err("write block")?; - } else { - println!( - "Failed to find block {} for {}", - bhash, - path.to_str().unwrap_or("not utf8?") - ); - } - } - } else { - println!( - "Failed to find blocklist {} for {}", - blh, - path.to_str().unwrap() - ); - } - } - } - } - _ => (), - } - Ok(()) - } } #[derive(Deserialize)] diff --git a/src/database.rs b/src/database.rs index 5fe5bbb..9f59322 100644 --- a/src/database.rs +++ b/src/database.rs @@ -23,6 +23,8 @@ use std::sync::Mutex; use zip; use zip::ZipArchive; +use crate::blockhash::BlockIdHash; + #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future struct Manifest { @@ -59,6 +61,7 @@ pub struct BlockLocation { } pub struct ZipArchiveWrapper { + pub zip_path: Arc, archive: ZipArchive, } @@ -75,17 +78,14 @@ impl HashToPath { } } - pub fn get_zip_path_by_base64(&self, block_id: &str) -> Option { - let key = base64::decode_config(block_id, base64::STANDARD).ok()?; - let key = SmallVec::from(key); - - self.hash2path.get(&key).map(|v| v.zip_path.path.clone()) + pub fn get_zip_path_by_block_id(&self, block_id: &BlockIdHash) -> Option { + self.hash2path + .get(&block_id.hash) + .map(|v| v.zip_path.path.clone()) } - pub fn get_location_by_base64(&self, block_id: &str) -> Option { - let key = base64::decode_config(block_id, base64::STANDARD).ok()?; - let key = SmallVec::from(key); - self.hash2path.get(&key).map(|v| v.clone()) + pub fn get_location_by_block_id(&self, block_id: &BlockIdHash) -> Option { + self.hash2path.get(&block_id.hash).map(|v| v.clone()) } } pub struct HashToBlocks { @@ -117,10 +117,20 @@ impl HashToBlocks { // } // } - pub fn get_location_by_base64(&self, block_id: &str) -> Option { + pub fn get_location_by_block_id(&self, block_id: &BlockIdHash) -> Option { if let Some(hash2path) = &self.hash2path { - hash2path.get_location_by_base64(block_id) + hash2path.get_location_by_block_id(block_id) } else { + let buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(buf); + for ziparch in self.zip2ziparchive.values() { + if let Some(index) = ziparch.archive.get_file_index(&name_reencoded) { + return Some(BlockLocation { + file_index: index as u32, + zip_path: ziparch.zip_path.clone(), + }); + } + } None } } @@ -131,15 +141,20 @@ impl HashToBlocks { zip.map(|zip| zip.archive.clone()) } - pub fn get_zip_by_block_id(&self, block_id: &str) -> Option> { + pub fn get_zip_by_block_id( + &self, + block_id: &BlockIdHash, + ) -> Option> { if let Some(hash2path) = &self.hash2path { - let zname = hash2path.get_zip_path_by_base64(block_id); + let zname = hash2path.get_zip_path_by_block_id(block_id); let zname = zname.map(|n| n.to_string_lossy().to_string()); let zipa = zname.map(|zname| self.get_zip_archive(&zname)).flatten(); zipa } else { + let buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(buf); for ziparch in self.zip2ziparchive.values() { - if ziparch.archive.contains_file_name(block_id) { + if ziparch.archive.contains_file_name(&name_reencoded) { return Some(ziparch.archive.clone()); } } @@ -148,19 +163,19 @@ impl HashToBlocks { } } -pub struct DB { +pub struct DFileDatabase { inner: Arc>, manifest: Manifest, } -impl DB { - pub fn new(manifest_bytes: &[u8], use_hash_to_path: bool) -> Result { +impl DFileDatabase { + pub fn new(manifest_bytes: &[u8], use_hash_to_path: bool) -> Result { // let conn = UnQLite::create(file); // conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; let manifest: Manifest = serde_json::from_slice(manifest_bytes)?; let inner = Arc::new(Mutex::new(HashToBlocks::new(use_hash_to_path))); - let db = DB { inner, manifest }; + let db = Self { inner, manifest }; Ok(db) } @@ -293,11 +308,13 @@ impl DB { } } { - config - .buf_capacity - .store(32 * 1024, std::sync::atomic::Ordering::Relaxed); + use std::sync::atomic::Ordering; + config.buf_capacity.store(32 * 1024, Ordering::Relaxed); let mut inner = self.inner.lock().unwrap(); - let wrapper = ZipArchiveWrapper { archive: zip }; + let wrapper = ZipArchiveWrapper { + zip_path: arc_zippath.clone(), + archive: zip, + }; let path_str = arc_zippath.path_str.clone(); inner.zip2ziparchive.insert(path_str, wrapper); } @@ -325,8 +342,11 @@ impl DB { Ok(()) } - pub fn get_block_id_location(&self, block_id: &str) -> Option { - self.inner.lock().unwrap().get_location_by_base64(block_id) + pub fn get_block_id_location(&self, block_id: &BlockIdHash) -> Option { + self.inner + .lock() + .unwrap() + .get_location_by_block_id(block_id) } // pub fn get_zip_path_from_block_id(&self, block_id: &str) -> Option { @@ -338,22 +358,22 @@ impl DB { // } - pub fn get_zip_by_block_id(&self, block_id: &str) -> Option> { + pub fn get_zip_by_block_id( + &self, + block_id: &BlockIdHash, + ) -> Option> { self.inner.lock().unwrap().get_zip_by_block_id(block_id) } - pub fn get_content_block(&self, block_id: &str) -> Result>> { + pub fn get_content_block(&self, block_id: &BlockIdHash) -> Result>> { let mut output = Vec::new(); - let name_reencoded: String = base64::encode_config( - &base64::decode(block_id).expect("wrong base64 block_id"), - base64::URL_SAFE, - ); - //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); - let ziparch = self.get_zip_by_block_id(&name_reencoded); + let ziparch = self.get_zip_by_block_id(&block_id); if let Some(mut ziparch) = ziparch { + let buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(buf); let mut block = ziparch .by_name(&name_reencoded) .wrap_err("block file by name not found even though we indexed it before")?; diff --git a/src/hexdisplay.rs b/src/hexdisplay.rs new file mode 100644 index 0000000..a6221a2 --- /dev/null +++ b/src/hexdisplay.rs @@ -0,0 +1,60 @@ +pub struct HexDisplayBytes<'a>(pub &'a [u8]); +impl<'a> std::fmt::Display for HexDisplayBytes<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for byte in self.0.as_ref().iter() { + let (high, low) = byte2hex(*byte, HEX_CHARS_LOWER); + + write!(f, "{}{}", high as char, low as char)?; + } + + Ok(()) + } +} + +pub struct EscapeWholeString<'a>(pub &'a [u8]); +impl<'a> std::fmt::Display for EscapeWholeString<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for byte in self.0.as_ref().iter() { + let (high, low) = byte2hex(*byte, HEX_CHARS_LOWER); + + write!(f, "\\x{}{}", high as char, low as char)?; + } + + Ok(()) + } +} + +pub struct EscapeRawString<'a>(pub &'a [u8]); +impl<'a> std::fmt::Display for EscapeRawString<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "\"")?; + for &b in self.0.as_ref().iter() { + escape_byte_maybe(f, b)?; + } + write!(f, "\"")?; + + Ok(()) + } +} + +fn escape_byte_maybe(f: &mut std::fmt::Formatter<'_>, b: u8) -> std::fmt::Result { + if b > 32 && b < 126 && b != '"' as u8 { + write!(f, "{}", b as char)?; + } else { + let (high, low) = byte2hex(b, HEX_CHARS_LOWER); + + write!(f, "\\x{}{}", high as char, low as char)?; + } + Ok(()) +} + +const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef"; +//const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF"; + +// the inverse of `val`. +fn byte2hex(byte: u8, table: &[u8; 16]) -> (u8, u8) { + let high = table[((byte & 0xf0) >> 4) as usize]; + let low = table[(byte & 0x0f) as usize]; + + (high, low) +} diff --git a/src/main.rs b/src/main.rs index 82c10e8..012fb39 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,9 @@ +mod blockhash; mod blockid; mod database; +mod hexdisplay; +mod restoring; +mod sorting; mod stripbom; use blockid::*; @@ -10,6 +14,7 @@ use eyre::{Context, Result}; use num_cpus; use pbr::ProgressBar; use rayon::prelude::*; +use sorting::compare_fileentry; use std::fs; use std::fs::File; use std::io::{BufReader, Read}; @@ -17,6 +22,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use zip; +use crate::restoring::restore_file; use crate::stripbom::StripBom; #[derive(Parser)] @@ -169,9 +175,13 @@ fn run() -> Result<()> { // Open dblock db connection and build db println!(); println!("Indexing dblocks"); - let dblock_db = DB::new(&manifest_contents, args.hash_to_path)?; + let dblock_db = DFileDatabase::new(&manifest_contents, args.hash_to_path)?; dblock_db.create_block_id_to_filenames(&zip_file_names)?; + println!("Sorting file_entries"); + + sort_files_sequentially(&mut file_entries, &dblock_db); + let show_progress = args.progress_bar.unwrap_or_default(); println!("Restoring directory structure"); let mut pb = if show_progress { @@ -180,19 +190,14 @@ fn run() -> Result<()> { None }; - for d in file_entries.iter().filter(|f| f.is_folder()) { - d.restore_file(&dblock_db, &restore_dir) - .wrap_err("restoring dir")?; + for entry in file_entries.iter().filter(|f| f.is_folder()) { + restore_file(entry, &dblock_db, &restore_dir).wrap_err("restoring dir")?; if let Some(pb) = &mut pb { pb.inc(); } } println!(); - println!("Sorting file_entries"); - - sort_files_sequentially(&mut file_entries, &dblock_db); - println!("Restoring files"); let pb = if show_progress { Some(Arc::new(Mutex::new(ProgressBar::new(file_count as u64)))) @@ -205,9 +210,8 @@ fn run() -> Result<()> { .filter(|f| f.is_file()) .par_bridge() //.iter() - .try_for_each(|f| -> Result<()> { - f.restore_file(&dblock_db, &restore_dir) - .wrap_err("restoring file entry")?; + .try_for_each(|entry_file| -> Result<()> { + restore_file(entry_file, &dblock_db, &restore_dir).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } @@ -219,8 +223,8 @@ fn run() -> Result<()> { /// Not necessary, but useful to speed up file reads from HDD /// from like 200 Mbit/s to 700 Mbit/s -fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DB) { - file_entries.sort_by(|a, b| a.compare(b, dblock_db)); +fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DFileDatabase) { + file_entries.sort_by(|a, b| compare_fileentry(a, b, dblock_db)); } #[cfg(feature = "dhat-heap")] diff --git a/src/restoring.rs b/src/restoring.rs new file mode 100644 index 0000000..7fdae2f --- /dev/null +++ b/src/restoring.rs @@ -0,0 +1,102 @@ +use std::{ + fs::{self, File}, + io::{Seek, SeekFrom, Write}, + path::Path, +}; + +use crate::{ + blockhash::BlockIdHash, + blockid::{FileEntry, FileType}, + database::DFileDatabase, +}; +use eyre::eyre; +use eyre::{Context, Result}; + +pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) -> Result<()> { + let root_path = Path::new(restore_path); + let dfile_path = &entry.path[0..]; + let dfile_path = dfile_path.replacen(":\\", "\\", 1); + let dfile_path = dfile_path.replace("\\", "/"); + let relative_file_path = Path::new(&dfile_path); + + let path = Path::join(root_path, relative_file_path); + + match &entry.file_type { + FileType::Folder { .. } => { + fs::create_dir_all(path)?; + } + FileType::File { hash, size, .. } => { + // Small files only have one block + if entry.block_lists.is_empty() { + let loc = db.get_block_id_location(hash); + println!( + "restoring file (single) {:?}, index:{:?}", + relative_file_path, + loc.map(|loc| loc.file_index) + ); + + let mut out_file = File::create(path.clone())?; + let block = db.get_content_block(hash)?; + if let Some(block) = block { + out_file + .write_all(block.as_ref()) + .wrap_err("write single-block file")?; + } else if *size > 0 { + println!( + "Missing block {:?} for {}", + hash, + path.to_str().unwrap_or("not utf8?") + ); + } + } else { + let loc = entry + .block_lists + .first() + .map(|hash| db.get_block_id_location(hash)) + .flatten(); + println!( + "restoring file (blocks) {:?}, index:{:?}", + relative_file_path, + loc.map(|loc| loc.file_index) + ); + let mut out_file = File::create(path.clone())?; + // Each blockid points to a list of blockids + for (blhi, blh) in entry.block_lists.iter().enumerate() { + let blockhashoffset = blhi * db.offset_size(); + let binary_hashes = db.get_content_block(blh)?; + if let Some(binary_hashes) = binary_hashes { + for (bi, bhash) in binary_hashes.chunks(db.hash_size()).enumerate() { + //let bhash = base64::encode(bhash); + let bhash = BlockIdHash::from_bytes(bhash) + .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; + let block = db.get_content_block(&bhash)?; + + if let Some(block) = block { + out_file + .seek(SeekFrom::Start( + (blockhashoffset + bi * db.block_size()) as u64, + )) + .wrap_err("seek blockhashoffset + bi * db.block_size()")?; + out_file.write_all(&block).wrap_err("write block")?; + } else { + println!( + "Failed to find block {} for {}", + bhash, + path.to_str().unwrap_or("not utf8?") + ); + } + } + } else { + println!( + "Failed to find blocklist {} for {}", + blh, + path.to_str().unwrap() + ); + } + } + } + } + _ => (), + } + Ok(()) +} diff --git a/src/sorting.rs b/src/sorting.rs new file mode 100644 index 0000000..6c69818 --- /dev/null +++ b/src/sorting.rs @@ -0,0 +1,30 @@ +use std::cmp::Ordering; + +use crate::{ + blockid::{FileEntry, FileType}, + database::{BlockLocation, DFileDatabase}, +}; + +/// Optional. Used for sorting. +pub fn get_first_bytes_location(entry: &FileEntry, db: &DFileDatabase) -> Option { + match &entry.file_type { + FileType::File { hash, .. } => { + if entry.block_lists.is_empty() { + db.get_block_id_location(hash) + } else { + let first = entry.block_lists.first(); + + first.map(|bid| db.get_block_id_location(bid)).flatten() + } + } + _ => None, + } +} + +/// Optional. Used for sorting. +pub fn compare_fileentry(entry_a: &FileEntry, entry_b: &FileEntry, db: &DFileDatabase) -> Ordering { + let a = get_first_bytes_location(entry_a, db); + let b = get_first_bytes_location(entry_b, db); + + a.cmp(&b).then_with(|| entry_a.cmp(entry_b)) +} From 6ff3608e46c8b79acacb8c70902474d1a3deb985 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:58:43 +0100 Subject: [PATCH 06/38] more refactor into .rs files --- src/database.rs | 206 ++-------------------------------------------- src/main.rs | 1 + src/sorting.rs | 3 +- src/ziparchive.rs | 105 +++++++++++++++++++++++ 4 files changed, 117 insertions(+), 198 deletions(-) create mode 100644 src/ziparchive.rs diff --git a/src/database.rs b/src/database.rs index 9f59322..24a50ac 100644 --- a/src/database.rs +++ b/src/database.rs @@ -8,13 +8,7 @@ use serde::Deserialize; use serde_json; use smallvec::SmallVec; use std::collections::HashMap; -use std::fs::File; -use std::io::BufRead; -use std::io::BufReader; -use std::io::IoSliceMut; use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; use std::path::Path; use std::path::PathBuf; use std::sync::atomic::AtomicU32; @@ -24,6 +18,11 @@ use zip; use zip::ZipArchive; use crate::blockhash::BlockIdHash; +use crate::ziparchive::BlockLocation; +use crate::ziparchive::MyCloneFileConfig; +use crate::ziparchive::MyCloneFileReader; +use crate::ziparchive::ZipArchiveWrapper; +use crate::ziparchive::ZipLocation; #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future @@ -44,27 +43,6 @@ struct Manifest { pub(self) app_version: String, } -#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] -/// Path to dblock.zip -pub struct ZipLocation { - pub path_str: String, - pub path: PathBuf, -} - -#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] -pub struct BlockLocation { - /// Which file inside the zip - pub file_index: u32, - - /// Which dblock.zip file - pub zip_path: Arc, -} - -pub struct ZipArchiveWrapper { - pub zip_path: Arc, - archive: ZipArchive, -} - pub struct HashToPath { /// Maps hash (without base64) to location in dblock.zip /// @@ -189,9 +167,10 @@ impl DFileDatabase { ) .progress_chars("##-"), ); - paths - .par_iter() - .try_for_each(|zip_path| self.import_from_zip(zip_path))?; + paths.par_iter().try_for_each(|zip_path| { + self.import_from_zip(zip_path) + .wrap_err_with(|| format!("import_from_zip: {:?}", zip_path)) + })?; Ok(()) } @@ -206,8 +185,6 @@ impl DFileDatabase { let zipbuf = MyCloneFileReader::new(config.clone())?; let zip = zip::ZipArchive::new(zipbuf)?; - let zip_len = zip.len(); - let arc_zippath = Arc::new(ZipLocation { path: zip_path.clone(), path_str: zip_path.to_string_lossy().to_string(), @@ -236,77 +213,6 @@ impl DFileDatabase { } } - if false { - let mut hvec = Vec::new(); - let (sender, receiver) = crossbeam_channel::bounded(zip_len + 1); - - // make workers - for _t in 0..16 { - //println!("Make worker {}", t); - - let receiver = receiver.clone(); // clone for this thread - - //let zip_path = zip_path.clone(); - let inner = self.inner.clone(); - let mut zip = zip.clone(); - let arc_zippath = arc_zippath.clone(); - let handler = std::thread::spawn(move || { - let mut progress_burst = 0; - - loop { - let r = receiver.recv(); - match r { - Ok(file_index) => { - //let s = rng.gen_range(100..1000); - - //thread::sleep(Duration::from_millis(s)); - - let hash_path = - zip.by_index(file_index).unwrap().name().to_string(); - let hash = - base64::decode_config(&hash_path, base64::URL_SAFE).unwrap(); - { - let mut inner = inner.lock().unwrap(); - - if let Some(hash2path) = &mut inner.hash2path { - hash2path.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_zippath.clone(), - file_index: file_index as u32, - }, - ); - } - } - - progress_burst += 1; - if progress_burst > 1000 { - //pb.inc(progress_burst); - println!( - "found hash {}/{} in {:?}", - file_index, zip_len, arc_zippath.path - ); - progress_burst = 0; - } - } - _ => break, - } - } - //pb.inc(progress_burst); - }); - - hvec.push(handler); - } - - for i in 0..zip_len { - sender.send(i).unwrap(); - } - drop(sender); - - for h in hvec { - h.join().unwrap(); - } - } { use std::sync::atomic::Ordering; config.buf_capacity.store(32 * 1024, Ordering::Relaxed); @@ -318,27 +224,7 @@ impl DFileDatabase { let path_str = arc_zippath.path_str.clone(); inner.zip2ziparchive.insert(path_str, wrapper); } - // let paths: Vec = (0..zip_len) - // .into_par_iter() - // .map_init( - // || { - - // zip - // }, - // |zip, i| , - // ) - // .collect(); - - // let bytes = zippath.as_bytes(); - // for p in paths { - - // //conn.kv_store(hash, bytes).unwrap(); - - // //println!("len: {}", hash.len()); - // //println!("zippath:{} hash:{}", zippath, p); - // } - //conn.commit().unwrap(); Ok(()) } @@ -401,77 +287,3 @@ impl DFileDatabase { 32 } } -pub struct MyCloneFileConfig { - pub path: PathBuf, - /// Changes after the files are indexed. - /// Bigger buf helps with large file reads. - /// Smaller buf does less redundant byte reads from disk when indexing. - pub buf_capacity: AtomicU32, -} -pub struct MyCloneFileReader { - pub config: Arc, - buf_reader: BufReader, -} - -impl Clone for MyCloneFileReader { - fn clone(&self) -> Self { - Self::new(self.config.clone()).unwrap() - } -} - -impl MyCloneFileReader { - pub fn new(config: Arc) -> Result { - let target_file = File::open(&config.path)?; - let cap = config - .buf_capacity - .load(std::sync::atomic::Ordering::Relaxed); - let filebuf = BufReader::with_capacity(cap as usize, target_file); - - Ok(Self { - config: config.clone(), - buf_reader: filebuf, - }) - } -} - -impl Read for MyCloneFileReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.buf_reader.read(buf) - } - - fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { - self.buf_reader.read_exact(buf) - } - - fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> std::io::Result { - self.buf_reader.read_vectored(bufs) - } - - fn read_to_end(&mut self, buf: &mut Vec) -> std::io::Result { - self.buf_reader.read_to_end(buf) - } - - fn read_to_string(&mut self, buf: &mut String) -> std::io::Result { - self.buf_reader.read_to_string(buf) - } -} - -impl Seek for MyCloneFileReader { - fn seek(&mut self, pos: SeekFrom) -> std::io::Result { - self.buf_reader.seek(pos) - } - - fn stream_position(&mut self) -> std::io::Result { - self.buf_reader.stream_position() - } -} - -impl BufRead for MyCloneFileReader { - fn fill_buf(&mut self) -> std::io::Result<&[u8]> { - self.buf_reader.fill_buf() - } - - fn consume(&mut self, amt: usize) { - self.buf_reader.consume(amt) - } -} diff --git a/src/main.rs b/src/main.rs index 012fb39..54f2ad0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ mod hexdisplay; mod restoring; mod sorting; mod stripbom; +mod ziparchive; use blockid::*; use clap::Parser; diff --git a/src/sorting.rs b/src/sorting.rs index 6c69818..c3a77c0 100644 --- a/src/sorting.rs +++ b/src/sorting.rs @@ -2,7 +2,8 @@ use std::cmp::Ordering; use crate::{ blockid::{FileEntry, FileType}, - database::{BlockLocation, DFileDatabase}, + database::DFileDatabase, + ziparchive::BlockLocation, }; /// Optional. Used for sorting. diff --git a/src/ziparchive.rs b/src/ziparchive.rs new file mode 100644 index 0000000..0487082 --- /dev/null +++ b/src/ziparchive.rs @@ -0,0 +1,105 @@ +use std::{ + fs::File, + io::{BufRead, BufReader, IoSliceMut, Read, Seek, SeekFrom}, + path::PathBuf, + sync::{atomic::AtomicU32, Arc}, +}; + +use eyre::Result; +use zip::ZipArchive; + +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +/// Path to dblock.zip +pub struct ZipLocation { + pub path_str: String, + pub path: PathBuf, +} + +#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] +pub struct BlockLocation { + /// Which file inside the zip + pub file_index: u32, + + /// Which dblock.zip file + pub zip_path: Arc, +} + +pub struct ZipArchiveWrapper { + pub zip_path: Arc, + pub archive: ZipArchive, +} + +pub struct MyCloneFileConfig { + pub path: PathBuf, + /// Changes after the files are indexed. + /// Bigger buf helps with large file reads. + /// Smaller buf does less redundant byte reads from disk when indexing. + pub buf_capacity: AtomicU32, +} +pub struct MyCloneFileReader { + pub config: Arc, + buf_reader: BufReader, +} + +impl Clone for MyCloneFileReader { + fn clone(&self) -> Self { + Self::new(self.config.clone()).unwrap() + } +} + +impl MyCloneFileReader { + pub fn new(config: Arc) -> Result { + let target_file = File::open(&config.path)?; + let cap = config + .buf_capacity + .load(std::sync::atomic::Ordering::Relaxed); + let filebuf = BufReader::with_capacity(cap as usize, target_file); + + Ok(Self { + config: config.clone(), + buf_reader: filebuf, + }) + } +} + +impl Read for MyCloneFileReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.buf_reader.read(buf) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { + self.buf_reader.read_exact(buf) + } + + fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> std::io::Result { + self.buf_reader.read_vectored(bufs) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> std::io::Result { + self.buf_reader.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> std::io::Result { + self.buf_reader.read_to_string(buf) + } +} + +impl Seek for MyCloneFileReader { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.buf_reader.seek(pos) + } + + fn stream_position(&mut self) -> std::io::Result { + self.buf_reader.stream_position() + } +} + +impl BufRead for MyCloneFileReader { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + self.buf_reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.buf_reader.consume(amt) + } +} From de1c375442c972ac089292dc9b9f95c5d351f004 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:35:09 +0100 Subject: [PATCH 07/38] add sha2 + refactor restoring --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 8 +- src/restoring.rs | 204 ++++++++++++++++++++++++++++++++--------------- src/sorting.rs | 6 ++ 5 files changed, 150 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9129060..2ac4fe6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,6 +1054,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", + "sha2", "smallvec", "unqlite", "zip", diff --git a/Cargo.toml b/Cargo.toml index b819f97..3d8566f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,3 +29,4 @@ serde_path_to_error = "0.1" smallvec = "*" crossbeam-channel = "*" dhat = "*" +sha2 = "0.10.6" diff --git a/src/main.rs b/src/main.rs index 54f2ad0..ab842e7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,6 @@ use eyre::{Context, Result}; use num_cpus; use pbr::ProgressBar; use rayon::prelude::*; -use sorting::compare_fileentry; use std::fs; use std::fs::File; use std::io::{BufReader, Read}; @@ -24,6 +23,7 @@ use std::sync::{Arc, Mutex}; use zip; use crate::restoring::restore_file; +use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; #[derive(Parser)] @@ -222,12 +222,6 @@ fn run() -> Result<()> { Ok(()) } -/// Not necessary, but useful to speed up file reads from HDD -/// from like 200 Mbit/s to 700 Mbit/s -fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DFileDatabase) { - file_entries.sort_by(|a, b| compare_fileentry(a, b, dblock_db)); -} - #[cfg(feature = "dhat-heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; diff --git a/src/restoring.rs b/src/restoring.rs index 7fdae2f..af2bbf2 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -1,16 +1,32 @@ use std::{ + cell::RefCell, fs::{self, File}, io::{Seek, SeekFrom, Write}, - path::Path, + path::{Path, PathBuf}, }; use crate::{ blockhash::BlockIdHash, blockid::{FileEntry, FileType}, database::DFileDatabase, + hexdisplay::HexDisplayBytes, }; use eyre::eyre; use eyre::{Context, Result}; +use sha2::{Digest, Sha256}; + +struct RestoreFileContext<'a> { + db: &'a DFileDatabase, + + entry: &'a FileEntry, + hash: &'a BlockIdHash, + size: i64, + + debug_location: bool, + hasher: RefCell>, + path: PathBuf, + relative_file_path: PathBuf, +} pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) -> Result<()> { let root_path = Path::new(restore_path); @@ -26,77 +42,139 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - fs::create_dir_all(path)?; } FileType::File { hash, size, .. } => { + let hasher = Sha256::new(); + let context = RestoreFileContext { + entry, + db, + debug_location: false, + hash, + size: *size, + hasher: RefCell::new(Some(hasher)), + path: path.to_path_buf(), + relative_file_path: relative_file_path.to_path_buf(), + }; + // Small files only have one block if entry.block_lists.is_empty() { - let loc = db.get_block_id_location(hash); - println!( - "restoring file (single) {:?}, index:{:?}", - relative_file_path, - loc.map(|loc| loc.file_index) - ); - - let mut out_file = File::create(path.clone())?; - let block = db.get_content_block(hash)?; + restore_file_singleblock(&context)?; + } else { + restore_file_multiblock(&context)?; + } + + check_file_hash(&context)?; + } + _ => (), + } + Ok(()) +} + +fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { + if ctx.debug_location { + let loc = ctx.db.get_block_id_location(ctx.hash); + println!( + "restoring file (single) {:?}, index:{:?}", + ctx.relative_file_path, + loc.map(|loc| loc.file_index) + ); + } + + let mut out_file = File::create(ctx.path.clone())?; + let block = ctx.db.get_content_block(ctx.hash)?; + if let Some(block) = block { + out_file + .write_all(block.as_ref()) + .wrap_err("write single-block file")?; + + { + let mut hasher = ctx.hasher.borrow_mut(); + if let Some(h) = hasher.as_mut() { + h.update(block.as_slice()); + } + } + } else if ctx.size > 0 { + println!("Missing block {} for {:?}", ctx.hash, ctx.path,); + } + Ok(()) +} + +fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { + if ctx.debug_location { + let loc = ctx + .entry + .block_lists + .first() + .map(|hash| ctx.db.get_block_id_location(hash)) + .flatten(); + println!( + "restoring file (blocks) {:?}, index:{:?}", + ctx.relative_file_path, + loc.map(|loc| loc.file_index) + ); + } + let mut out_file = File::create(ctx.path.clone())?; + // Each blockid points to a list of blockids + for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { + let blockhashoffset = blhi * ctx.db.offset_size(); + let binary_hashes = ctx + .db + .get_content_block(blh) + .wrap_err_with(|| format!("get main content block: {}", blh))?; + if let Some(binary_hashes) = binary_hashes { + for (bi, bhash) in binary_hashes.chunks(ctx.db.hash_size()).enumerate() { + //let bhash = base64::encode(bhash); + let bhash = BlockIdHash::from_bytes(bhash) + .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; + let block = ctx.db.get_content_block(&bhash).wrap_err_with(|| { + format!("get one of content blocks (number {}): {}", bi, blh) + })?; + if let Some(block) = block { + let offset = (blockhashoffset + bi * ctx.db.block_size()) as u64; out_file - .write_all(block.as_ref()) - .wrap_err("write single-block file")?; - } else if *size > 0 { - println!( - "Missing block {:?} for {}", - hash, - path.to_str().unwrap_or("not utf8?") - ); - } - } else { - let loc = entry - .block_lists - .first() - .map(|hash| db.get_block_id_location(hash)) - .flatten(); - println!( - "restoring file (blocks) {:?}, index:{:?}", - relative_file_path, - loc.map(|loc| loc.file_index) - ); - let mut out_file = File::create(path.clone())?; - // Each blockid points to a list of blockids - for (blhi, blh) in entry.block_lists.iter().enumerate() { - let blockhashoffset = blhi * db.offset_size(); - let binary_hashes = db.get_content_block(blh)?; - if let Some(binary_hashes) = binary_hashes { - for (bi, bhash) in binary_hashes.chunks(db.hash_size()).enumerate() { - //let bhash = base64::encode(bhash); - let bhash = BlockIdHash::from_bytes(bhash) - .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; - let block = db.get_content_block(&bhash)?; - - if let Some(block) = block { - out_file - .seek(SeekFrom::Start( - (blockhashoffset + bi * db.block_size()) as u64, - )) - .wrap_err("seek blockhashoffset + bi * db.block_size()")?; - out_file.write_all(&block).wrap_err("write block")?; - } else { - println!( - "Failed to find block {} for {}", - bhash, - path.to_str().unwrap_or("not utf8?") - ); - } + .seek(SeekFrom::Start(offset)) + .wrap_err("seek blockhashoffset + bi * db.block_size()")?; + out_file.write_all(&block).wrap_err("write block")?; + { + let mut hasher = ctx.hasher.borrow_mut(); + if let Some(h) = hasher.as_mut() { + h.update(block.as_slice()); } - } else { - println!( - "Failed to find blocklist {} for {}", - blh, - path.to_str().unwrap() - ); } + } else { + println!("Failed to find block {} for {:?}", bhash, ctx.path,); } } + } else { + println!("Failed to find blocklist {} for {:?}", blh, ctx.path,); + } + } + + Ok(()) +} + +fn check_file_hash<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { + let hasher = { + let mut hasher = None; + std::mem::swap(&mut hasher, &mut ctx.hasher.borrow_mut()); + + hasher + }; + if let Some(hasher) = hasher { + let calculated_hash: &[u8] = &hasher.finalize()[..]; + let expected_hash = ctx.hash.hash.as_slice(); + if expected_hash == calculated_hash { + println!( + "hash is valid {} == {}", + HexDisplayBytes(expected_hash), + HexDisplayBytes(calculated_hash) + ); + } else { + Err(eyre!( + "hash is invalid: expected != calculated, {} != {}", + HexDisplayBytes(expected_hash), + HexDisplayBytes(calculated_hash) + ))? } - _ => (), } Ok(()) } diff --git a/src/sorting.rs b/src/sorting.rs index c3a77c0..4a2f35c 100644 --- a/src/sorting.rs +++ b/src/sorting.rs @@ -6,6 +6,12 @@ use crate::{ ziparchive::BlockLocation, }; +/// Not necessary, but useful to speed up file reads from HDD +/// from like 200 Mbit/s to 700 Mbit/s +pub fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DFileDatabase) { + file_entries.sort_by(|a, b| compare_fileentry(a, b, dblock_db)); +} + /// Optional. Used for sorting. pub fn get_first_bytes_location(entry: &FileEntry, db: &DFileDatabase) -> Option { match &entry.file_type { From 22856bd1aa3abf29917bb7ae6e6a814613ddae85 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 14:37:07 +0100 Subject: [PATCH 08/38] remove allocations --- src/restoring.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/restoring.rs b/src/restoring.rs index af2bbf2..8af2a83 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -2,7 +2,7 @@ use std::{ cell::RefCell, fs::{self, File}, io::{Seek, SeekFrom, Write}, - path::{Path, PathBuf}, + path::Path, }; use crate::{ @@ -24,8 +24,8 @@ struct RestoreFileContext<'a> { debug_location: bool, hasher: RefCell>, - path: PathBuf, - relative_file_path: PathBuf, + absolute_path: &'a Path, + relative_file_path: &'a Path, } pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) -> Result<()> { @@ -50,8 +50,8 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - hash, size: *size, hasher: RefCell::new(Some(hasher)), - path: path.to_path_buf(), - relative_file_path: relative_file_path.to_path_buf(), + absolute_path: &path, + relative_file_path: &relative_file_path, }; // Small files only have one block @@ -78,7 +78,7 @@ fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { ); } - let mut out_file = File::create(ctx.path.clone())?; + let mut out_file = File::create(ctx.absolute_path.clone())?; let block = ctx.db.get_content_block(ctx.hash)?; if let Some(block) = block { out_file @@ -92,7 +92,7 @@ fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { } } } else if ctx.size > 0 { - println!("Missing block {} for {:?}", ctx.hash, ctx.path,); + println!("Missing block {} for {:?}", ctx.hash, ctx.absolute_path,); } Ok(()) } @@ -111,7 +111,7 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { loc.map(|loc| loc.file_index) ); } - let mut out_file = File::create(ctx.path.clone())?; + let mut out_file = File::create(ctx.absolute_path.clone())?; // Each blockid points to a list of blockids for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { let blockhashoffset = blhi * ctx.db.offset_size(); @@ -141,11 +141,14 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { } } } else { - println!("Failed to find block {} for {:?}", bhash, ctx.path,); + println!("Failed to find block {} for {:?}", bhash, ctx.absolute_path,); } } } else { - println!("Failed to find blocklist {} for {:?}", blh, ctx.path,); + println!( + "Failed to find blocklist {} for {:?}", + blh, ctx.absolute_path, + ); } } From 63ede5aade14fe4cac8e71007b858b2a4f622fc3 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:03:26 +0100 Subject: [PATCH 09/38] disable hasher for 0-size files --- src/restoring.rs | 60 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/src/restoring.rs b/src/restoring.rs index 8af2a83..4260280 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -23,6 +23,7 @@ struct RestoreFileContext<'a> { size: i64, debug_location: bool, + strict_block_size: bool, hasher: RefCell>, absolute_path: &'a Path, relative_file_path: &'a Path, @@ -42,14 +43,15 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - fs::create_dir_all(path)?; } FileType::File { hash, size, .. } => { - let hasher = Sha256::new(); + let hasher = if *size > 0 { Some(Sha256::new()) } else { None }; let context = RestoreFileContext { entry, db, debug_location: false, + strict_block_size: true, hash, size: *size, - hasher: RefCell::new(Some(hasher)), + hasher: RefCell::new(hasher), absolute_path: &path, relative_file_path: &relative_file_path, }; @@ -92,7 +94,11 @@ fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { } } } else if ctx.size > 0 { - println!("Missing block {} for {:?}", ctx.hash, ctx.absolute_path,); + Err(eyre!( + "Missing block {} for {:?}", + ctx.hash, + ctx.absolute_path + ))?; } Ok(()) } @@ -120,6 +126,7 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { .get_content_block(blh) .wrap_err_with(|| format!("get main content block: {}", blh))?; if let Some(binary_hashes) = binary_hashes { + let mut last_block_size = None; for (bi, bhash) in binary_hashes.chunks(ctx.db.hash_size()).enumerate() { //let bhash = base64::encode(bhash); let bhash = BlockIdHash::from_bytes(bhash) @@ -129,10 +136,12 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { })?; if let Some(block) = block { - let offset = (blockhashoffset + bi * ctx.db.block_size()) as u64; + let full_block = ctx.db.block_size(); + + let offset = (blockhashoffset + bi * full_block) as u64; out_file .seek(SeekFrom::Start(offset)) - .wrap_err("seek blockhashoffset + bi * db.block_size()")?; + .wrap_err("seek blockhashoffset + bi * full_block")?; out_file.write_all(&block).wrap_err("write block")?; { let mut hasher = ctx.hasher.borrow_mut(); @@ -140,15 +149,32 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { h.update(block.as_slice()); } } + if ctx.strict_block_size { + if let Some(last) = last_block_size { + if last != full_block { + Err(eyre!( + "last block size != full_block, {} != {}", + last, + full_block + ))?; + } + } + last_block_size = Some(block.len()); + } } else { - println!("Failed to find block {} for {:?}", bhash, ctx.absolute_path,); + Err(eyre!( + "Failed to find block {} for {:?}", + bhash, + ctx.absolute_path + ))?; } } } else { - println!( + Err(eyre!( "Failed to find blocklist {} for {:?}", - blh, ctx.absolute_path, - ); + blh, + ctx.absolute_path, + ))?; } } @@ -156,6 +182,9 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { } fn check_file_hash<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { + if ctx.size == 0 { + return Ok(()); + } let hasher = { let mut hasher = None; std::mem::swap(&mut hasher, &mut ctx.hasher.borrow_mut()); @@ -166,11 +195,14 @@ fn check_file_hash<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { let calculated_hash: &[u8] = &hasher.finalize()[..]; let expected_hash = ctx.hash.hash.as_slice(); if expected_hash == calculated_hash { - println!( - "hash is valid {} == {}", - HexDisplayBytes(expected_hash), - HexDisplayBytes(calculated_hash) - ); + let debug_hash = false; + if debug_hash { + println!( + "hash is valid {} == {}", + HexDisplayBytes(expected_hash), + HexDisplayBytes(calculated_hash) + ); + } } else { Err(eyre!( "hash is invalid: expected != calculated, {} != {}", From 6ff1c329d098e88e9c59bdd032fb529bc669fdeb Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:06:08 +0100 Subject: [PATCH 10/38] cargo fmt --- src/blockid.rs | 4 +--- src/database.rs | 13 ++++++------- src/hexdisplay.rs | 3 +-- src/main.rs | 7 +++---- src/restoring.rs | 13 ++++++------- src/sorting.rs | 3 +-- src/ziparchive.rs | 3 +-- 7 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/blockid.rs b/src/blockid.rs index a4880aa..21ad8e7 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -1,14 +1,12 @@ use crate::blockhash::BlockIdHash; use crate::stripbom::strip_bom_from_bufread; +use eyre::eyre; use eyre::Context; use eyre::Result; use serde::Deserialize; use serde_json; use serde_json::de::IoRead; use serde_json::Deserializer; - -use eyre::eyre; - use std::io::prelude::*; #[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] diff --git a/src/database.rs b/src/database.rs index 24a50ac..04ded44 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,3 +1,9 @@ +use crate::blockhash::BlockIdHash; +use crate::ziparchive::BlockLocation; +use crate::ziparchive::MyCloneFileConfig; +use crate::ziparchive::MyCloneFileReader; +use crate::ziparchive::ZipArchiveWrapper; +use crate::ziparchive::ZipLocation; use base64; use eyre::Context; use eyre::Result; @@ -17,13 +23,6 @@ use std::sync::Mutex; use zip; use zip::ZipArchive; -use crate::blockhash::BlockIdHash; -use crate::ziparchive::BlockLocation; -use crate::ziparchive::MyCloneFileConfig; -use crate::ziparchive::MyCloneFileReader; -use crate::ziparchive::ZipArchiveWrapper; -use crate::ziparchive::ZipLocation; - #[derive(Deserialize)] #[allow(dead_code)] // Will use all these fields in the future struct Manifest { diff --git a/src/hexdisplay.rs b/src/hexdisplay.rs index a6221a2..f31b2b2 100644 --- a/src/hexdisplay.rs +++ b/src/hexdisplay.rs @@ -49,9 +49,8 @@ fn escape_byte_maybe(f: &mut std::fmt::Formatter<'_>, b: u8) -> std::fmt::Result } const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef"; -//const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF"; -// the inverse of `val`. +/// returns 2 chars representing byte in hex fn byte2hex(byte: u8, table: &[u8; 16]) -> (u8, u8) { let high = table[((byte & 0xf0) >> 4) as usize]; let low = table[(byte & 0x0f) as usize]; diff --git a/src/main.rs b/src/main.rs index ab842e7..1a11c66 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,9 @@ mod sorting; mod stripbom; mod ziparchive; +use crate::restoring::restore_file; +use crate::sorting::sort_files_sequentially; +use crate::stripbom::StripBom; use blockid::*; use clap::Parser; use database::*; @@ -22,10 +25,6 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use zip; -use crate::restoring::restore_file; -use crate::sorting::sort_files_sequentially; -use crate::stripbom::StripBom; - #[derive(Parser)] #[command(author, version, about, long_about = None)] struct CliArgs { diff --git a/src/restoring.rs b/src/restoring.rs index 4260280..528ce49 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -1,10 +1,3 @@ -use std::{ - cell::RefCell, - fs::{self, File}, - io::{Seek, SeekFrom, Write}, - path::Path, -}; - use crate::{ blockhash::BlockIdHash, blockid::{FileEntry, FileType}, @@ -14,6 +7,12 @@ use crate::{ use eyre::eyre; use eyre::{Context, Result}; use sha2::{Digest, Sha256}; +use std::{ + cell::RefCell, + fs::{self, File}, + io::{Seek, SeekFrom, Write}, + path::Path, +}; struct RestoreFileContext<'a> { db: &'a DFileDatabase, diff --git a/src/sorting.rs b/src/sorting.rs index 4a2f35c..45f8504 100644 --- a/src/sorting.rs +++ b/src/sorting.rs @@ -1,10 +1,9 @@ -use std::cmp::Ordering; - use crate::{ blockid::{FileEntry, FileType}, database::DFileDatabase, ziparchive::BlockLocation, }; +use std::cmp::Ordering; /// Not necessary, but useful to speed up file reads from HDD /// from like 200 Mbit/s to 700 Mbit/s diff --git a/src/ziparchive.rs b/src/ziparchive.rs index 0487082..7b3e1cd 100644 --- a/src/ziparchive.rs +++ b/src/ziparchive.rs @@ -1,11 +1,10 @@ +use eyre::Result; use std::{ fs::File, io::{BufRead, BufReader, IoSliceMut, Read, Seek, SeekFrom}, path::PathBuf, sync::{atomic::AtomicU32, Arc}, }; - -use eyre::Result; use zip::ZipArchive; #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] From 4d3a438c5d2d4df6cbf4368e62419e11dc894cec Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:20:10 +0100 Subject: [PATCH 11/38] cleanup Cargo.toml --- .gitignore | 1 + Cargo.lock | 1527 --------------------------------------------------- Cargo.toml | 14 +- src/main.rs | 21 +- 4 files changed, 16 insertions(+), 1547 deletions(-) delete mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index b9c6d36..15c764b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target **/*.rs.bk **/*.py +/Cargo.lock /debug_*.json /testout diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 2ac4fe6..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,1527 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" -dependencies = [ - "cfg-if 1.0.0", - "cipher", - "cpufeatures", - "opaque-debug", -] - -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "backtrace" -version = "0.3.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" -dependencies = [ - "byteorder", -] - -[[package]] -name = "base64ct" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" - -[[package]] -name = "bindgen" -version = "0.55.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443" -dependencies = [ - "bitflags", - "cexpr", - "cfg-if 0.1.10", - "clang-sys", - "clap 2.34.0", - "env_logger", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "which", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "block-buffer" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cc" -version = "1.0.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cexpr" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-integer", - "num-traits", - "time 0.1.45", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - -[[package]] -name = "clang-sys" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", -] - -[[package]] -name = "clap" -version = "4.0.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" -dependencies = [ - "bitflags", - "clap_derive", - "clap_lex", - "is-terminal", - "once_cell", - "strsim 0.10.0", - "termcolor", -] - -[[package]] -name = "clap_derive" -version = "4.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - -[[package]] -name = "console" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys", -] - -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - -[[package]] -name = "cpufeatures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" -dependencies = [ - "autocfg", - "cfg-if 1.0.0", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "cxx" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "dhat" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3" -dependencies = [ - "backtrace", - "lazy_static", - "mintex", - "parking_lot", - "rustc-hash", - "serde", - "serde_json", - "thousands", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "either" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "eyre" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" -dependencies = [ - "indenter", - "once_cell", -] - -[[package]] -name = "flate2" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "generic-array" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "gimli" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "heck" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" -dependencies = [ - "cxx", - "cxx-build", -] - -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - -[[package]] -name = "indicatif" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c60da1c9abea75996b70a931bba6c750730399005b61ccd853cee50ef3d0d0c" -dependencies = [ - "console", - "lazy_static", - "number_prefix", - "parking_lot", - "regex", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "is-terminal" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" -dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", - "rustix", - "windows-sys", -] - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" - -[[package]] -name = "jobserver" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.139" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if 1.0.0", - "winapi", -] - -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - -[[package]] -name = "linux-raw-sys" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - -[[package]] -name = "mintex" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb" -dependencies = [ - "once_cell", - "sys-info", -] - -[[package]] -name = "nom" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" -dependencies = [ - "memchr", - "version_check", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf9993e59c894e3c08aa1c2712914e9e6bf1fcbfc6bef283e2183df345a4fee" -dependencies = [ - "num-traits", -] - -[[package]] -name = "object" -version = "0.30.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8c786513eb403643f2a88c244c2aaa270ef2153f55094587d0c48a3cf22a83" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", -] - -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - -[[package]] -name = "paste" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - -[[package]] -name = "pbr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff5751d87f7c00ae6403eb1fcbba229b9c76c9a30de8c1cf87182177b168cea2" -dependencies = [ - "crossbeam-channel", - "libc", - "time 0.1.45", - "winapi", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quote" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" - -[[package]] -name = "rayon" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "rust-duplicati-restore" -version = "0.0.2" -dependencies = [ - "base64", - "chrono", - "clap 4.0.32", - "crossbeam-channel", - "dhat", - "eyre", - "indicatif", - "num_cpus", - "pbr", - "rayon", - "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "smallvec", - "unqlite", - "zip", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustix" -version = "0.36.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "ryu" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" - -[[package]] -name = "serde" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" -dependencies = [ - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sys-info" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "termcolor" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "thousands" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" - -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" -dependencies = [ - "itoa", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "time-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" -dependencies = [ - "time-core", -] - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "unicode-ident" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" - -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "unqlite" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae3a9e0f84985f05127d7f923ff129718a4db3585794e636540e109746dce452" -dependencies = [ - "bindgen", - "cc", - "libc", - "paste", -] - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - -[[package]] -name = "wasm-bindgen" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" -dependencies = [ - "cfg-if 1.0.0", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" - -[[package]] -name = "which" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" -dependencies = [ - "libc", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" - -[[package]] -name = "zip" -version = "0.6.3" -dependencies = [ - "aes", - "byteorder", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1", - "time 0.3.17", - "zstd", -] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.5+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index 3d8566f..f3a3e91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,18 +2,16 @@ name = "rust-duplicati-restore" description = "A program to restore duplicati backups" license = "MIT" -version = "0.0.2" -authors = ["Nathan McCarty "] -edition = "2018" +version = "0.0.3" +authors = ["Nathan McCarty ", "7ERr0r"] +edition = "2021" [features] dhat-heap = [] # if you are doing heap profiling -dhat-ad-hoc = [] # if you are doing ad hoc profiling [dependencies] -zip = { version = "*", path = "../zip" } -#zip = "*" +zip = { version = "*", git = "https://github.com/7ERr0r/zip-duplicati", rev = "77f115763e7d1e686273589e7b26f4efd3f5bf38" } chrono = "0.4.0" base64 = "0.10.1" pbr = "1.0.1" @@ -27,6 +25,6 @@ clap = { version = "4.0.32", features = ["derive"] } eyre = "0.6.8" serde_path_to_error = "0.1" smallvec = "*" -crossbeam-channel = "*" -dhat = "*" sha2 = "0.10.6" +crossbeam-channel = "0.5.6" +dhat = "0.3.2" diff --git a/src/main.rs b/src/main.rs index 1a11c66..f07e13a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,5 @@ +#![warn(rust_2018_idioms)] + mod blockhash; mod blockid; mod database; @@ -15,7 +17,6 @@ use clap::Parser; use database::*; use eyre::eyre; use eyre::{Context, Result}; -use num_cpus; use pbr::ProgressBar; use rayon::prelude::*; use std::fs; @@ -36,11 +37,11 @@ struct CliArgs { #[arg(short, long, value_name = "FILE")] restore_dir: String, - #[arg(short, long)] - cpu_count: Option, + #[arg(short, long, default_value_t = 4)] + rayon_threads: usize, #[arg(short, long)] - progress_bar: Option, + progress_bar: bool, /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. #[arg(short, long)] @@ -121,14 +122,11 @@ fn run() -> Result<()> { let backup_dir = args.backup_dir.trim(); let restore_dir = args.restore_dir.trim(); - // let db_location = Path::join(Path::new(restore_dir), Path::new("index.db")); - // let db_location = db_location.to_str().unwrap(); - - let cpu_count: usize = args.cpu_count.unwrap_or_else(|| num_cpus::get()); + let rayon_threads: usize = args.rayon_threads; // Set CPU count rayon::ThreadPoolBuilder::new() - .num_threads(cpu_count.min(4)) + .num_threads(rayon_threads) .build_global() .unwrap(); @@ -182,9 +180,8 @@ fn run() -> Result<()> { sort_files_sequentially(&mut file_entries, &dblock_db); - let show_progress = args.progress_bar.unwrap_or_default(); println!("Restoring directory structure"); - let mut pb = if show_progress { + let mut pb = if args.progress_bar { Some(ProgressBar::new(folder_count as u64)) } else { None @@ -199,7 +196,7 @@ fn run() -> Result<()> { println!(); println!("Restoring files"); - let pb = if show_progress { + let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new(file_count as u64)))) } else { None From 7efb787d8d8f361d6dae0f70944e7cd8aec9446c Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:28:58 +0100 Subject: [PATCH 12/38] fix clap args --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index f07e13a..4b00ba0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,13 +38,13 @@ struct CliArgs { restore_dir: String, #[arg(short, long, default_value_t = 4)] - rayon_threads: usize, + threads_rayon: usize, #[arg(short, long)] progress_bar: bool, /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. - #[arg(short, long)] + #[arg(long)] hash_to_path: bool, } @@ -122,7 +122,7 @@ fn run() -> Result<()> { let backup_dir = args.backup_dir.trim(); let restore_dir = args.restore_dir.trim(); - let rayon_threads: usize = args.rayon_threads; + let rayon_threads: usize = args.threads_rayon; // Set CPU count rayon::ThreadPoolBuilder::new() From 4a99c374dc0d6c05b72109f5939ec047161de812 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:29:09 +0100 Subject: [PATCH 13/38] update readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd9a29c..c1b67eb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,10 @@ Uses rayon to process files across many threads, to maximze restore speed. ## Getting Started Simply run the rust-duplicati-restore from the commandline. -It doesn't accept any flags and will prompt you for all information. + +``` +Usage: cargo run -- --backup-dir --restore-dir +``` ### Prerequisites @@ -24,7 +27,6 @@ Or download the latest binary from the artifacts ## Limitations -* Currently does not verify restored files * Does not yet support encrypted backups, I reccomend combining aescrypt with gnu parallel for decryption * Does not support remote repositories yet, I reccomend using rclone to pull donw a local copy @@ -44,3 +46,7 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md * Ben Fisher - His python script included in the Duplicati reposistory inspired this project, and this project was roughly based on it. + +* Nathan McCarty - Created Rust-Duplicati-Restore itself + +* 7ERr0r - Optimized ZIP reader. Added sha2 verification. From 24c0b0fcc564abd152127c00c8e2956ad4b45e06 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:31:12 +0100 Subject: [PATCH 14/38] dev as fast as --release --- Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index f3a3e91..5376319 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ version = "0.0.3" authors = ["Nathan McCarty ", "7ERr0r"] edition = "2021" +[profile.dev] +opt-level = 2 [features] dhat-heap = [] # if you are doing heap profiling From 9cd2359d74763fc51e037502e9dbefe1c9f8d073 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:35:31 +0100 Subject: [PATCH 15/38] add github workflow --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76ec346 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +# This workflow run tests and build for each push + +on: + push: + branches: + - main + - master + +jobs: + + test_phaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Update local toolchain + run: | + rustup update + rustup component add clippy + rustup install nightly + + - name: Toolchain info + run: | + cargo --version --verbose + rustc --version + cargo clippy --version + + - name: Lint + run: | + cd phaser + cargo fmt -- --check + cargo clippy -- -D warnings + + - name: Test + run: | + cd phaser + cargo check + cargo test --all + + - name: Build + run: | + cd phaser + cargo build --release \ No newline at end of file From 3ec9add839167f088bcb1a7123c9fbfaa9aa8131 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:48:51 +0100 Subject: [PATCH 16/38] fix clippy --- src/blockhash.rs | 2 +- src/blockid.rs | 19 ++++++------------- src/database.rs | 17 +++++++---------- src/hexdisplay.rs | 2 +- src/main.rs | 20 +++++++++++--------- src/restoring.rs | 17 ++++++++--------- src/sorting.rs | 4 ++-- src/stripbom.rs | 16 ++++++++-------- src/ziparchive.rs | 2 +- 9 files changed, 45 insertions(+), 54 deletions(-) diff --git a/src/blockhash.rs b/src/blockhash.rs index 15ac5fd..bc6b5cd 100644 --- a/src/blockhash.rs +++ b/src/blockhash.rs @@ -43,7 +43,7 @@ impl BlockIdHash { assert!(block_id_str.len() < buffer.capacity()); base64::decode_config_buf(block_id_str, config, buffer).ok()?; let hash = BlockIdHash { - hash: SmallVec::from_slice(&buffer), + hash: SmallVec::from_slice(buffer), }; buffer.clear(); Some(hash) diff --git a/src/blockid.rs b/src/blockid.rs index 21ad8e7..db47e8f 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -4,7 +4,6 @@ use eyre::eyre; use eyre::Context; use eyre::Result; use serde::Deserialize; -use serde_json; use serde_json::de::IoRead; use serde_json::Deserializer; use std::io::prelude::*; @@ -24,10 +23,7 @@ pub enum FileType { impl FileType { pub fn is_file(&self) -> bool { - match self { - FileType::File { .. } => true, - _ => false, - } + matches!(self, FileType::File { .. }) } #[allow(unused)] @@ -39,10 +35,7 @@ impl FileType { } pub fn is_folder(&self) -> bool { - match self { - FileType::Folder { .. } => true, - _ => false, - } + matches!(self, FileType::Folder { .. }) } } @@ -67,7 +60,7 @@ impl FileEntry { if let Some(blocks) = &ientry.blocklists { for block in blocks { block_lists.push( - BlockIdHash::from_base64(&block) + BlockIdHash::from_base64(block) .ok_or_else(|| eyre!("blocklists BlockIdHash::from_base64 fail"))?, ); } @@ -78,11 +71,11 @@ impl FileEntry { .hash .as_ref() .map(|hash| { - BlockIdHash::from_base64(&hash) + BlockIdHash::from_base64(hash) .ok_or_else(|| eyre!("ientry.hash BlockIdHash::from_base64 fail")) }) .ok_or_else(|| eyre!("hash not found"))??, - size: ientry.size.clone().ok_or_else(|| eyre!("size not found"))?, + size: ientry.size.ok_or_else(|| eyre!("size not found"))?, time: ientry.time.clone().ok_or_else(|| eyre!("time not found"))?, }, "Folder" => FileType::Folder { @@ -139,7 +132,7 @@ pub fn parse_dlist(dlist: &[u8]) -> Result> { /// Accepts the dlist as a Read trait /// Returns a Vec of FileEntrys -pub fn parse_dlist_read<'a, R: BufRead>(mut rdr: R) -> Result> { +pub fn parse_dlist_read(mut rdr: R) -> Result> { let mut file_entries = Vec::new(); strip_bom_from_bufread(&mut rdr)?; diff --git a/src/database.rs b/src/database.rs index 04ded44..c69f1c2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -4,14 +4,13 @@ use crate::ziparchive::MyCloneFileConfig; use crate::ziparchive::MyCloneFileReader; use crate::ziparchive::ZipArchiveWrapper; use crate::ziparchive::ZipLocation; -use base64; use eyre::Context; use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::IntoParallelRefIterator; use rayon::prelude::ParallelIterator; use serde::Deserialize; -use serde_json; + use smallvec::SmallVec; use std::collections::HashMap; use std::io::Read; @@ -20,7 +19,6 @@ use std::path::PathBuf; use std::sync::atomic::AtomicU32; use std::sync::Arc; use std::sync::Mutex; -use zip; use zip::ZipArchive; #[derive(Deserialize)] @@ -62,7 +60,7 @@ impl HashToPath { } pub fn get_location_by_block_id(&self, block_id: &BlockIdHash) -> Option { - self.hash2path.get(&block_id.hash).map(|v| v.clone()) + self.hash2path.get(&block_id.hash).cloned() } } pub struct HashToBlocks { @@ -101,7 +99,7 @@ impl HashToBlocks { let buf = &mut [0u8; 48]; let name_reencoded = block_id.as_base64_urlsafe(buf); for ziparch in self.zip2ziparchive.values() { - if let Some(index) = ziparch.archive.get_file_index(&name_reencoded) { + if let Some(index) = ziparch.archive.get_file_index(name_reencoded) { return Some(BlockLocation { file_index: index as u32, zip_path: ziparch.zip_path.clone(), @@ -125,13 +123,12 @@ impl HashToBlocks { if let Some(hash2path) = &self.hash2path { let zname = hash2path.get_zip_path_by_block_id(block_id); let zname = zname.map(|n| n.to_string_lossy().to_string()); - let zipa = zname.map(|zname| self.get_zip_archive(&zname)).flatten(); - zipa + zname.and_then(|zname| self.get_zip_archive(&zname)) } else { let buf = &mut [0u8; 48]; let name_reencoded = block_id.as_base64_urlsafe(buf); for ziparch in self.zip2ziparchive.values() { - if ziparch.archive.contains_file_name(&name_reencoded) { + if ziparch.archive.contains_file_name(name_reencoded) { return Some(ziparch.archive.clone()); } } @@ -254,13 +251,13 @@ impl DFileDatabase { let mut output = Vec::new(); //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); - let ziparch = self.get_zip_by_block_id(&block_id); + let ziparch = self.get_zip_by_block_id(block_id); if let Some(mut ziparch) = ziparch { let buf = &mut [0u8; 48]; let name_reencoded = block_id.as_base64_urlsafe(buf); let mut block = ziparch - .by_name(&name_reencoded) + .by_name(name_reencoded) .wrap_err("block file by name not found even though we indexed it before")?; block .read_to_end(&mut output) diff --git a/src/hexdisplay.rs b/src/hexdisplay.rs index f31b2b2..2655c71 100644 --- a/src/hexdisplay.rs +++ b/src/hexdisplay.rs @@ -38,7 +38,7 @@ impl<'a> std::fmt::Display for EscapeRawString<'a> { } fn escape_byte_maybe(f: &mut std::fmt::Formatter<'_>, b: u8) -> std::fmt::Result { - if b > 32 && b < 126 && b != '"' as u8 { + if b > 32 && b < 126 && b != b'"' { write!(f, "{}", b as char)?; } else { let (high, low) = byte2hex(b, HEX_CHARS_LOWER); diff --git a/src/main.rs b/src/main.rs index 4b00ba0..2b4d912 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,7 +24,6 @@ use std::fs::File; use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; -use zip; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -37,9 +36,11 @@ struct CliArgs { #[arg(short, long, value_name = "FILE")] restore_dir: String, + /// 1 thread will save and read files sequentially #[arg(short, long, default_value_t = 4)] threads_rayon: usize, + /// displays progress bar in CLI #[arg(short, long)] progress_bar: bool, @@ -62,15 +63,16 @@ fn main() { Err(err) => { println!("err: {:?}", err); } - Ok(_) => {} + Ok(_) => { + println!("Finished without errors!"); + } } } fn filename_ends_with>(path: P, suffix: &str) -> bool { path.as_ref() .file_name() - .map(|name| name.to_str()) - .flatten() + .and_then(|name| name.to_str()) .map(|name| name.ends_with(suffix)) .unwrap_or(false) } @@ -131,11 +133,11 @@ fn run() -> Result<()> { .unwrap(); // Find newest dlist - let mut dlist_file_paths: Vec = fs::read_dir(&backup_dir)? + let mut dlist_file_paths: Vec = fs::read_dir(backup_dir)? .filter_map(Result::ok) .filter(|f| path_is_dlist_zip(f.path())) //.map(|f| (f.metadata().map(|m| m.modified()), f)) - .map(|f| f.path().to_path_buf()) + .map(|f| f.path()) .collect(); dlist_file_paths.sort(); @@ -165,7 +167,7 @@ fn run() -> Result<()> { .unwrap() .filter_map(Result::ok) .filter(|f| path_is_dblock_zip(f.path())) - .map(|f| f.path().to_path_buf()) + .map(|f| f.path()) .collect(); println!("Found {} dblocks", zip_file_names.len()); @@ -188,7 +190,7 @@ fn run() -> Result<()> { }; for entry in file_entries.iter().filter(|f| f.is_folder()) { - restore_file(entry, &dblock_db, &restore_dir).wrap_err("restoring dir")?; + restore_file(entry, &dblock_db, restore_dir).wrap_err("restoring dir")?; if let Some(pb) = &mut pb { pb.inc(); } @@ -208,7 +210,7 @@ fn run() -> Result<()> { .par_bridge() //.iter() .try_for_each(|entry_file| -> Result<()> { - restore_file(entry_file, &dblock_db, &restore_dir).wrap_err("restoring file entry")?; + restore_file(entry_file, &dblock_db, restore_dir).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } diff --git a/src/restoring.rs b/src/restoring.rs index 528ce49..d78ebd3 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -32,7 +32,7 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - let root_path = Path::new(restore_path); let dfile_path = &entry.path[0..]; let dfile_path = dfile_path.replacen(":\\", "\\", 1); - let dfile_path = dfile_path.replace("\\", "/"); + let dfile_path = dfile_path.replace('\\', "/"); let relative_file_path = Path::new(&dfile_path); let path = Path::join(root_path, relative_file_path); @@ -52,7 +52,7 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - size: *size, hasher: RefCell::new(hasher), absolute_path: &path, - relative_file_path: &relative_file_path, + relative_file_path, }; // Small files only have one block @@ -69,7 +69,7 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - Ok(()) } -fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { +fn restore_file_singleblock(ctx: &RestoreFileContext<'_>) -> Result<()> { if ctx.debug_location { let loc = ctx.db.get_block_id_location(ctx.hash); println!( @@ -79,7 +79,7 @@ fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { ); } - let mut out_file = File::create(ctx.absolute_path.clone())?; + let mut out_file = File::create(ctx.absolute_path)?; let block = ctx.db.get_content_block(ctx.hash)?; if let Some(block) = block { out_file @@ -102,21 +102,20 @@ fn restore_file_singleblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { Ok(()) } -fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { +fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { if ctx.debug_location { let loc = ctx .entry .block_lists .first() - .map(|hash| ctx.db.get_block_id_location(hash)) - .flatten(); + .and_then(|hash| ctx.db.get_block_id_location(hash)); println!( "restoring file (blocks) {:?}, index:{:?}", ctx.relative_file_path, loc.map(|loc| loc.file_index) ); } - let mut out_file = File::create(ctx.absolute_path.clone())?; + let mut out_file = File::create(ctx.absolute_path)?; // Each blockid points to a list of blockids for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { let blockhashoffset = blhi * ctx.db.offset_size(); @@ -180,7 +179,7 @@ fn restore_file_multiblock<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { Ok(()) } -fn check_file_hash<'a>(ctx: &RestoreFileContext<'a>) -> Result<()> { +fn check_file_hash(ctx: &RestoreFileContext<'_>) -> Result<()> { if ctx.size == 0 { return Ok(()); } diff --git a/src/sorting.rs b/src/sorting.rs index 45f8504..431b1e2 100644 --- a/src/sorting.rs +++ b/src/sorting.rs @@ -7,7 +7,7 @@ use std::cmp::Ordering; /// Not necessary, but useful to speed up file reads from HDD /// from like 200 Mbit/s to 700 Mbit/s -pub fn sort_files_sequentially(file_entries: &mut Vec, dblock_db: &DFileDatabase) { +pub fn sort_files_sequentially(file_entries: &mut [FileEntry], dblock_db: &DFileDatabase) { file_entries.sort_by(|a, b| compare_fileentry(a, b, dblock_db)); } @@ -20,7 +20,7 @@ pub fn get_first_bytes_location(entry: &FileEntry, db: &DFileDatabase) -> Option } else { let first = entry.block_lists.first(); - first.map(|bid| db.get_block_id_location(bid)).flatten() + first.and_then(|bid| db.get_block_id_location(bid)) } } _ => None, diff --git a/src/stripbom.rs b/src/stripbom.rs index 73dd3ce..51fa4dc 100644 --- a/src/stripbom.rs +++ b/src/stripbom.rs @@ -7,17 +7,17 @@ pub trait StripBom { impl StripBom for str { fn strip_bom(&self) -> &str { - if self.starts_with("\u{feff}") { - &self[3..] + if let Some(stripped) = self.strip_prefix('\u{feff}') { + stripped } else { - &self[..] + self } } } impl StripBom for String { fn strip_bom(&self) -> &str { - &self[..].strip_bom() + self[..].strip_bom() } } @@ -30,11 +30,11 @@ pub trait StripBomBytes { } impl StripBomBytes for [u8] { - fn strip_bom<'a>(&'a self) -> &'a [u8] { - if starts_with_bom_bytes(self) { - &self[3..] + fn strip_bom(&self) -> &[u8] { + if let Some(stripped) = self.strip_prefix(&[0xEF, 0xBB, 0xBF]) { + stripped } else { - &self[..] + self } } } diff --git a/src/ziparchive.rs b/src/ziparchive.rs index 7b3e1cd..02e05fa 100644 --- a/src/ziparchive.rs +++ b/src/ziparchive.rs @@ -55,7 +55,7 @@ impl MyCloneFileReader { let filebuf = BufReader::with_capacity(cap as usize, target_file); Ok(Self { - config: config.clone(), + config, buf_reader: filebuf, }) } From b1974f226ff8818f3490c4ab1f929768dd9045b4 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:49:51 +0100 Subject: [PATCH 17/38] fix workflow --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76ec346..7431348 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: jobs: - test_phaser: + all_duplicati_restore: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -19,27 +19,24 @@ jobs: run: | rustup update rustup component add clippy - rustup install nightly + rustup install nightly - name: Toolchain info run: | cargo --version --verbose rustc --version - cargo clippy --version + cargo clippy --version - name: Lint run: | - cd phaser cargo fmt -- --check - cargo clippy -- -D warnings + cargo clippy -- -D warnings - name: Test run: | - cd phaser cargo check - cargo test --all + cargo test --all - name: Build run: | - cd phaser - cargo build --release \ No newline at end of file + cargo build --release \ No newline at end of file From c9dd711068776dbf01b87e9ecead9fcdd80c4f7e Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:37:35 +0100 Subject: [PATCH 18/38] windows build --- .github/workflows/ci.yml | 3 ++- Cargo.toml | 10 ++++++---- src/main.rs | 15 ++++++++++++--- src/restoring.rs | 18 +++++++++++++----- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7431348..ce67b5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,4 +39,5 @@ jobs: - name: Build run: | - cargo build --release \ No newline at end of file + cargo build --release + cargo build --release --target x86_64-pc-windows-gnu \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 5376319..17fb96e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,8 @@ edition = "2021" opt-level = 2 [features] -dhat-heap = [] # if you are doing heap profiling +dhat-heap = ["dep:dhat"] # if you are doing heap profiling +unqlite = ["dep:unqlite"] [dependencies] zip = { version = "*", git = "https://github.com/7ERr0r/zip-duplicati", rev = "77f115763e7d1e686273589e7b26f4efd3f5bf38" } @@ -20,8 +21,8 @@ pbr = "1.0.1" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0.39" rayon = "1.0" -num_cpus = "1.10.0" -unqlite = "1.5" +#num_cpus = "1.10.0" +unqlite = { version = "1.5", optional = true } indicatif = "0.11.0" clap = { version = "4.0.32", features = ["derive"] } eyre = "0.6.8" @@ -29,4 +30,5 @@ serde_path_to_error = "0.1" smallvec = "*" sha2 = "0.10.6" crossbeam-channel = "0.5.6" -dhat = "0.3.2" +dhat = { version = "0.3.2", optional = true } + diff --git a/src/main.rs b/src/main.rs index 2b4d912..c8218d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod sorting; mod stripbom; mod ziparchive; -use crate::restoring::restore_file; +use crate::restoring::{restore_file, RestoreParams}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; use blockid::*; @@ -47,6 +47,10 @@ struct CliArgs { /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. #[arg(long)] hash_to_path: bool, + + /// true to restore windows backup on linux + #[arg(long)] + replace_backslash_to_slash: Option, } fn main() { @@ -188,9 +192,14 @@ fn run() -> Result<()> { } else { None }; + let restore_params = RestoreParams { + db: &dblock_db, + restore_path: restore_dir, + replace_backslash_to_slash: args.replace_backslash_to_slash.unwrap_or(!cfg!(windows)), + }; for entry in file_entries.iter().filter(|f| f.is_folder()) { - restore_file(entry, &dblock_db, restore_dir).wrap_err("restoring dir")?; + restore_file(entry, &restore_params).wrap_err("restoring dir")?; if let Some(pb) = &mut pb { pb.inc(); } @@ -210,7 +219,7 @@ fn run() -> Result<()> { .par_bridge() //.iter() .try_for_each(|entry_file| -> Result<()> { - restore_file(entry_file, &dblock_db, restore_dir).wrap_err("restoring file entry")?; + restore_file(entry_file, &restore_params).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } diff --git a/src/restoring.rs b/src/restoring.rs index d78ebd3..8870c47 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -28,11 +28,19 @@ struct RestoreFileContext<'a> { relative_file_path: &'a Path, } -pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) -> Result<()> { - let root_path = Path::new(restore_path); +pub struct RestoreParams<'a> { + pub db: &'a DFileDatabase, + pub restore_path: &'a str, + pub replace_backslash_to_slash: bool, +} + +pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> { + let root_path = Path::new(params.restore_path); let dfile_path = &entry.path[0..]; - let dfile_path = dfile_path.replacen(":\\", "\\", 1); - let dfile_path = dfile_path.replace('\\', "/"); + let mut dfile_path = dfile_path.replacen(":\\", "\\", 1); + if params.replace_backslash_to_slash { + dfile_path = dfile_path.replace('\\', "/"); + } let relative_file_path = Path::new(&dfile_path); let path = Path::join(root_path, relative_file_path); @@ -45,7 +53,7 @@ pub fn restore_file(entry: &FileEntry, db: &DFileDatabase, restore_path: &str) - let hasher = if *size > 0 { Some(Sha256::new()) } else { None }; let context = RestoreFileContext { entry, - db, + db: params.db, debug_location: false, strict_block_size: true, hash, From 64f06a5ca6da472bd1c051f3c486df92e47065f5 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:39:31 +0100 Subject: [PATCH 19/38] archive binary files --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce67b5c..e7d1b19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,4 +40,12 @@ jobs: - name: Build run: | cargo build --release - cargo build --release --target x86_64-pc-windows-gnu \ No newline at end of file + cargo build --release --target x86_64-pc-windows-gnu + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: target binaries + path: | + target/release/rust-duplicati-restore + target/x86_64-pc-windows-gnu/release/rust-duplicati-restore.exe \ No newline at end of file From c6e10a1a1ed147db9fab972899bc4b68f9fcc7dc Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:55:25 +0100 Subject: [PATCH 20/38] cleanup --- .github/workflows/ci.yml | 2 +- src/main.rs | 56 ++++++++++++++++++++++++++-------------- src/restoring.rs | 2 ++ 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7d1b19..3512d18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: run: | rustup update rustup component add clippy - rustup install nightly + rustup target add x86_64-pc-windows-gnu - name: Toolchain info run: | diff --git a/src/main.rs b/src/main.rs index c8218d9..a5c7500 100644 --- a/src/main.rs +++ b/src/main.rs @@ -128,11 +128,9 @@ fn run() -> Result<()> { let backup_dir = args.backup_dir.trim(); let restore_dir = args.restore_dir.trim(); - let rayon_threads: usize = args.threads_rayon; - // Set CPU count rayon::ThreadPoolBuilder::new() - .num_threads(rayon_threads) + .num_threads(args.threads_rayon) .build_global() .unwrap(); @@ -140,7 +138,6 @@ fn run() -> Result<()> { let mut dlist_file_paths: Vec = fs::read_dir(backup_dir)? .filter_map(Result::ok) .filter(|f| path_is_dlist_zip(f.path())) - //.map(|f| (f.metadata().map(|m| m.modified()), f)) .map(|f| f.path()) .collect(); @@ -186,40 +183,59 @@ fn run() -> Result<()> { sort_files_sequentially(&mut file_entries, &dblock_db); - println!("Restoring directory structure"); - let mut pb = if args.progress_bar { - Some(ProgressBar::new(folder_count as u64)) - } else { - None - }; let restore_params = RestoreParams { db: &dblock_db, restore_path: restore_dir, replace_backslash_to_slash: args.replace_backslash_to_slash.unwrap_or(!cfg!(windows)), + file_count, + folder_count, }; + restore_all(&args, &restore_params, &file_entries)?; - for entry in file_entries.iter().filter(|f| f.is_folder()) { - restore_file(entry, &restore_params).wrap_err("restoring dir")?; - if let Some(pb) = &mut pb { - pb.inc(); - } - } + Ok(()) +} + +fn restore_all( + args: &CliArgs, + params: &RestoreParams<'_>, + file_entries: &[FileEntry], +) -> Result<()> { + println!("Restoring directory structure"); + let pb = if args.progress_bar { + Some(Arc::new(Mutex::new(ProgressBar::new( + params.folder_count as u64, + )))) + } else { + None + }; + + file_entries + .iter() + .filter(|f| f.is_folder()) + .par_bridge() + .try_for_each(|entry_folder| -> Result<()> { + restore_file(entry_folder, params).wrap_err("restoring dir")?; + if let Some(pb) = &pb { + pb.lock().unwrap().inc(); + } + Ok(()) + })?; println!(); println!("Restoring files"); let pb = if args.progress_bar { - Some(Arc::new(Mutex::new(ProgressBar::new(file_count as u64)))) + Some(Arc::new(Mutex::new(ProgressBar::new( + params.file_count as u64, + )))) } else { None }; file_entries - //.par_iter() .iter() .filter(|f| f.is_file()) .par_bridge() - //.iter() .try_for_each(|entry_file| -> Result<()> { - restore_file(entry_file, &restore_params).wrap_err("restoring file entry")?; + restore_file(entry_file, params).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } diff --git a/src/restoring.rs b/src/restoring.rs index 8870c47..78a2771 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -32,6 +32,8 @@ pub struct RestoreParams<'a> { pub db: &'a DFileDatabase, pub restore_path: &'a str, pub replace_backslash_to_slash: bool, + pub file_count: usize, + pub folder_count: usize, } pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> { From 70ac81de1be1848a11b3806f49a3b57b87509d60 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 18:17:55 +0100 Subject: [PATCH 21/38] allow verification without write --- src/blockid.rs | 24 ++++++++++- src/main.rs | 108 +++++++++++++++++++++++++++++++++-------------- src/restoring.rs | 83 ++++++++++++++++++++++++------------ 3 files changed, 153 insertions(+), 62 deletions(-) diff --git a/src/blockid.rs b/src/blockid.rs index db47e8f..9f2020b 100644 --- a/src/blockid.rs +++ b/src/blockid.rs @@ -6,6 +6,7 @@ use eyre::Result; use serde::Deserialize; use serde_json::de::IoRead; use serde_json::Deserializer; +use smallvec::SmallVec; use std::io::prelude::*; #[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] @@ -47,7 +48,7 @@ pub struct FileEntry { #[allow(unused)] pub metasize: i64, pub file_type: FileType, - pub block_lists: Vec, + pub block_lists: SmallVec<[BlockIdHash; 1]>, } impl FileEntry { @@ -55,7 +56,7 @@ impl FileEntry { let path = ientry.path.clone(); let metahash = ientry.metahash.clone(); let metasize = ientry.metasize; - let mut block_lists = Vec::new(); + let mut block_lists = SmallVec::new(); if let Some(blocks) = &ientry.blocklists { for block in blocks { @@ -103,6 +104,25 @@ impl FileEntry { pub fn is_folder(&self) -> bool { self.file_type.is_folder() } + + /// How much bytes it probably takes on disk when restoring + pub fn predicted_time(&self) -> u64 { + // Not an accurate number + let psize = 8 * 1024 + self.path.len() as u64; + if let FileType::File { size, .. } = self.file_type { + psize + size as u64 + } else { + psize + } + } + + pub fn bytes_size(&self) -> u64 { + if let FileType::File { size, .. } = self.file_type { + size as u64 + } else { + 0 + } + } } #[derive(Deserialize)] diff --git a/src/main.rs b/src/main.rs index a5c7500..d5e85f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod sorting; mod stripbom; mod ziparchive; -use crate::restoring::{restore_file, RestoreParams}; +use crate::restoring::{restore_file, RestoreParams, RestoreSummary}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; use blockid::*; @@ -34,7 +34,7 @@ struct CliArgs { /// a location to restore to #[arg(short, long, value_name = "FILE")] - restore_dir: String, + restore_dir: Option, /// 1 thread will save and read files sequentially #[arg(short, long, default_value_t = 4)] @@ -51,6 +51,10 @@ struct CliArgs { /// true to restore windows backup on linux #[arg(long)] replace_backslash_to_slash: Option, + + /// true to verify without writing files to disk + #[arg(long)] + verify_only: bool, } fn main() { @@ -125,8 +129,16 @@ fn read_manifest>(dlist_path: P) -> Result> { fn run() -> Result<()> { let args = CliArgs::parse(); - let backup_dir = args.backup_dir.trim(); - let restore_dir = args.restore_dir.trim(); + let backup_dir = args.backup_dir.trim().to_string(); + let restore_dir = if !args.verify_only { + let dir = args + .restore_dir + .as_ref() + .ok_or_else(|| eyre!("--restore_dir not provided"))?; + Some(dir.trim()) + } else { + None + }; // Set CPU count rayon::ThreadPoolBuilder::new() @@ -135,7 +147,7 @@ fn run() -> Result<()> { .unwrap(); // Find newest dlist - let mut dlist_file_paths: Vec = fs::read_dir(backup_dir)? + let mut dlist_file_paths: Vec = fs::read_dir(&backup_dir)? .filter_map(Result::ok) .filter(|f| path_is_dlist_zip(f.path())) .map(|f| f.path()) @@ -151,44 +163,71 @@ fn run() -> Result<()> { "Newest: {:?} appears to be newest dlist, using it.", newest_dlist ); - println!("Parsing dlist"); - let mut file_entries = parse_dlist_file(newest_dlist)?; - println!("Parsing manifest"); let manifest_contents = read_manifest(newest_dlist)?; - let file_count = file_entries.iter().filter(|f| f.is_file()).count(); - println!("{} files to be restored", file_count); - let folder_count = file_entries.iter().filter(|f| f.is_folder()).count(); - println!("{} folders to be restored", folder_count); + // Open dblock db connection and build db println!(); + let db_join = std::thread::spawn(move || -> Result { + println!("Listing dblocks"); + // Get list of dblocks + let zip_file_names: Vec = fs::read_dir(backup_dir) + .wrap_err("read_dir(backup_dir)")? + .filter_map(Result::ok) + .filter(|f| path_is_dblock_zip(f.path())) + .map(|f| f.path()) + .collect(); + + println!("Found {} dblocks", zip_file_names.len()); + println!("Indexing dblocks"); + let dblock_db = DFileDatabase::new(&manifest_contents, args.hash_to_path)?; + dblock_db.create_block_id_to_filenames(&zip_file_names)?; + Ok(dblock_db) + }); - // Get list of dblocks - let zip_file_names: Vec = fs::read_dir(backup_dir) - .unwrap() - .filter_map(Result::ok) - .filter(|f| path_is_dblock_zip(f.path())) - .map(|f| f.path()) - .collect(); - - println!("Found {} dblocks", zip_file_names.len()); + println!("Parsing dlist"); + let file_entries = parse_dlist_file(newest_dlist)?; + let file_entries = Arc::new(file_entries); + + let entries = file_entries.clone(); + let summary_join = std::thread::spawn(move || { + let file_count = entries.iter().filter(|f| f.is_file()).count(); + let folder_count = entries.iter().filter(|f| f.is_folder()).count(); + let predicted_bytes: u64 = entries.iter().map(|f| f.predicted_time()).sum(); + let total_bytes: u64 = entries.iter().map(|f| f.bytes_size()).sum(); + RestoreSummary { + file_count, + folder_count, + total_bytes, + predicted_bytes, + } + }); - // Open dblock db connection and build db println!(); - println!("Indexing dblocks"); - let dblock_db = DFileDatabase::new(&manifest_contents, args.hash_to_path)?; - dblock_db.create_block_id_to_filenames(&zip_file_names)?; + + let summary = summary_join.join().unwrap(); + + println!("{} files to be restored", summary.file_count); + println!("{} folders to be restored", summary.folder_count); + println!("{} bytes in files", summary.total_bytes); + println!( + "{} bytes on drive to be restored (predicted)", + summary.predicted_bytes + ); + + let dblock_db = db_join.join().unwrap()?; println!("Sorting file_entries"); + let mut file_entries = Arc::try_unwrap(file_entries).expect("no other owners of Arc"); + sort_files_sequentially(&mut file_entries, &dblock_db); let restore_params = RestoreParams { db: &dblock_db, restore_path: restore_dir, replace_backslash_to_slash: args.replace_backslash_to_slash.unwrap_or(!cfg!(windows)), - file_count, - folder_count, + summary, }; restore_all(&args, &restore_params, &file_entries)?; @@ -200,10 +239,15 @@ fn restore_all( params: &RestoreParams<'_>, file_entries: &[FileEntry], ) -> Result<()> { - println!("Restoring directory structure"); + let doing = if params.restore_path.is_some() { + "Restoring" + } else { + "Verifying" + }; + println!("{} directory structure", doing); let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new( - params.folder_count as u64, + params.summary.folder_count as u64, )))) } else { None @@ -222,10 +266,10 @@ fn restore_all( })?; println!(); - println!("Restoring files"); + println!("{} files", doing); let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new( - params.file_count as u64, + params.summary.predicted_bytes, )))) } else { None @@ -237,7 +281,7 @@ fn restore_all( .try_for_each(|entry_file| -> Result<()> { restore_file(entry_file, params).wrap_err("restoring file entry")?; if let Some(pb) = &pb { - pb.lock().unwrap().inc(); + pb.lock().unwrap().add(entry_file.predicted_time()); } Ok(()) })?; diff --git a/src/restoring.rs b/src/restoring.rs index 78a2771..f1b1080 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -11,7 +11,7 @@ use std::{ cell::RefCell, fs::{self, File}, io::{Seek, SeekFrom, Write}, - path::Path, + path::{Path, PathBuf}, }; struct RestoreFileContext<'a> { @@ -24,32 +24,48 @@ struct RestoreFileContext<'a> { debug_location: bool, strict_block_size: bool, hasher: RefCell>, - absolute_path: &'a Path, - relative_file_path: &'a Path, + + /// None if only verifying + absolute_path: Option<&'a PathBuf>, + /// None if only verifying + relative_file_path: Option<&'a PathBuf>, +} + +pub struct RestoreSummary { + pub file_count: usize, + pub folder_count: usize, + pub total_bytes: u64, + pub predicted_bytes: u64, } pub struct RestoreParams<'a> { pub db: &'a DFileDatabase, - pub restore_path: &'a str, + pub restore_path: Option<&'a str>, pub replace_backslash_to_slash: bool, - pub file_count: usize, - pub folder_count: usize, + pub summary: RestoreSummary, } pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> { - let root_path = Path::new(params.restore_path); - let dfile_path = &entry.path[0..]; - let mut dfile_path = dfile_path.replacen(":\\", "\\", 1); - if params.replace_backslash_to_slash { - dfile_path = dfile_path.replace('\\', "/"); - } - let relative_file_path = Path::new(&dfile_path); + let (path, relative_file_path) = if let Some(restore_path) = ¶ms.restore_path { + let root_path = Path::new(restore_path); + let dfile_path = &entry.path[0..]; + let mut dfile_path = dfile_path.replacen(":\\", "\\", 1); + if params.replace_backslash_to_slash { + dfile_path = dfile_path.replace('\\', "/"); + } + let relative_file_path = PathBuf::from(&dfile_path); - let path = Path::join(root_path, relative_file_path); + let path = Path::join(root_path, &relative_file_path); + (Some(path), Some(relative_file_path)) + } else { + (None, None) + }; match &entry.file_type { FileType::Folder { .. } => { - fs::create_dir_all(path)?; + if let Some(path) = path { + fs::create_dir_all(path)?; + } } FileType::File { hash, size, .. } => { let hasher = if *size > 0 { Some(Sha256::new()) } else { None }; @@ -61,8 +77,8 @@ pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> hash, size: *size, hasher: RefCell::new(hasher), - absolute_path: &path, - relative_file_path, + absolute_path: path.as_ref(), + relative_file_path: relative_file_path.as_ref(), }; // Small files only have one block @@ -89,13 +105,18 @@ fn restore_file_singleblock(ctx: &RestoreFileContext<'_>) -> Result<()> { ); } - let mut out_file = File::create(ctx.absolute_path)?; + let mut out_file = if let Some(path) = ctx.absolute_path { + Some(File::create(path)?) + } else { + None + }; let block = ctx.db.get_content_block(ctx.hash)?; if let Some(block) = block { - out_file - .write_all(block.as_ref()) - .wrap_err("write single-block file")?; - + if let Some(out_file) = &mut out_file { + out_file + .write_all(block.as_ref()) + .wrap_err("write single-block file")?; + } { let mut hasher = ctx.hasher.borrow_mut(); if let Some(h) = hasher.as_mut() { @@ -125,7 +146,11 @@ fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { loc.map(|loc| loc.file_index) ); } - let mut out_file = File::create(ctx.absolute_path)?; + let mut out_file = if let Some(path) = ctx.absolute_path { + Some(File::create(path)?) + } else { + None + }; // Each blockid points to a list of blockids for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { let blockhashoffset = blhi * ctx.db.offset_size(); @@ -146,11 +171,13 @@ fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { if let Some(block) = block { let full_block = ctx.db.block_size(); - let offset = (blockhashoffset + bi * full_block) as u64; - out_file - .seek(SeekFrom::Start(offset)) - .wrap_err("seek blockhashoffset + bi * full_block")?; - out_file.write_all(&block).wrap_err("write block")?; + if let Some(out_file) = &mut out_file { + let offset = (blockhashoffset + bi * full_block) as u64; + out_file + .seek(SeekFrom::Start(offset)) + .wrap_err("seek blockhashoffset + bi * full_block")?; + out_file.write_all(&block).wrap_err("write (multi) block")?; + } { let mut hasher = ctx.hasher.borrow_mut(); if let Some(h) = hasher.as_mut() { From 96d6416af839692b8fde002c8e62aa2c40f55ce3 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 18:26:43 +0100 Subject: [PATCH 22/38] github workflow: add apt gcc-mingw-w64-x86-64 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3512d18..eb00e46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: rustup update rustup component add clippy rustup target add x86_64-pc-windows-gnu + apt-get install -y gcc-mingw-w64-x86-64 - name: Toolchain info run: | From 7e9bb9717fb32ebb4765ab7e7742c98d1b0e4d3e Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 18:34:44 +0100 Subject: [PATCH 23/38] github workflow: sudo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb00e46..38d7ec1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: rustup update rustup component add clippy rustup target add x86_64-pc-windows-gnu - apt-get install -y gcc-mingw-w64-x86-64 + sudo apt-get install -y gcc-mingw-w64-x86-64 - name: Toolchain info run: | From e78ee0ea2921f366f660408983134be62ada1027 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 18:47:06 +0100 Subject: [PATCH 24/38] workflow: rust.yml From https://github.com/nicolas-van/rust-cross-compile-example/blob/main/.github/workflows/rust.yml --- .github/workflows/{ci.yml => ci.yml.unused} | 0 .github/workflows/rust.yml | 143 ++++++++++++++++++++ 2 files changed, 143 insertions(+) rename .github/workflows/{ci.yml => ci.yml.unused} (100%) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml.unused similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yml.unused diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..14dca2a --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,143 @@ +name: Rust + +on: + push: +# pull_request: + +env: + CARGO_TERM_COLOR: always + +defaults: + run: + # necessary for windows + shell: bash + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Cargo cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ./target + key: test-cargo-registry + - name: List + run: find ./ + - name: Run tests + run: cargo test --verbose + + build: + strategy: + fail-fast: false + matrix: + # a list of all the targets + include: + - TARGET: x86_64-unknown-linux-gnu # tested in a debian container on a mac + OS: ubuntu-latest + - TARGET: x86_64-unknown-linux-musl # test in an alpine container on a mac + OS: ubuntu-latest + - TARGET: aarch64-unknown-linux-gnu # tested on aws t4g.nano + OS: ubuntu-latest + - TARGET: aarch64-unknown-linux-musl # tested on aws t4g.nano in alpine container + OS: ubuntu-latest + - TARGET: armv7-unknown-linux-gnueabihf # raspberry pi 2-3-4, not tested + OS: ubuntu-latest + - TARGET: armv7-unknown-linux-musleabihf # raspberry pi 2-3-4, not tested + OS: ubuntu-latest + - TARGET: arm-unknown-linux-gnueabihf # raspberry pi 0-1, not tested + OS: ubuntu-latest + - TARGET: arm-unknown-linux-musleabihf # raspberry pi 0-1, not tested + OS: ubuntu-latest + - TARGET: x86_64-apple-darwin # tested on a mac, is not properly signed so there are security warnings + OS: macos-latest + - TARGET: x86_64-pc-windows-msvc # tested on a windows machine + OS: windows-latest + needs: test + runs-on: ${{ matrix.OS }} + env: + NAME: rust-cross-compile-example # change with the name of your project + TARGET: ${{ matrix.TARGET }} + OS: ${{ matrix.OS }} + steps: + - uses: actions/checkout@v2 + - name: Cargo cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ./target + key: build-cargo-registry-${{matrix.TARGET}} + - name: List + run: find ./ + - name: Install and configure dependencies + run: | + # dependencies are only needed on ubuntu as that's the only place where + # we make cross-compilation + if [[ $OS =~ ^ubuntu.*$ ]]; then + sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf + fi + + # some additional configuration for cross-compilation on linux + cat >>~/.cargo/config < Date: Thu, 12 Jan 2023 18:55:41 +0100 Subject: [PATCH 25/38] workflow rust fix --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 14dca2a..1b7f344 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -58,7 +58,7 @@ jobs: needs: test runs-on: ${{ matrix.OS }} env: - NAME: rust-cross-compile-example # change with the name of your project + NAME: rust-duplicati-restore TARGET: ${{ matrix.TARGET }} OS: ${{ matrix.OS }} steps: @@ -77,7 +77,7 @@ jobs: # dependencies are only needed on ubuntu as that's the only place where # we make cross-compilation if [[ $OS =~ ^ubuntu.*$ ]]; then - sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf + sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf musl-gcc fi # some additional configuration for cross-compilation on linux From 7d3536adcf250481527743574420c6479e4bae36 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 19:00:18 +0100 Subject: [PATCH 26/38] apt install musl --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1b7f344..67c19a2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -77,7 +77,7 @@ jobs: # dependencies are only needed on ubuntu as that's the only place where # we make cross-compilation if [[ $OS =~ ^ubuntu.*$ ]]; then - sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf musl-gcc + sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf musl fi # some additional configuration for cross-compilation on linux From a6df2ab8113c5789fcb1f7e4b5d0e9efbcaab489 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 19:14:24 +0100 Subject: [PATCH 27/38] disable musl --- .github/workflows/rust.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 67c19a2..fb7dfae 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,20 +37,20 @@ jobs: include: - TARGET: x86_64-unknown-linux-gnu # tested in a debian container on a mac OS: ubuntu-latest - - TARGET: x86_64-unknown-linux-musl # test in an alpine container on a mac - OS: ubuntu-latest + # - TARGET: x86_64-unknown-linux-musl # test in an alpine container on a mac + # OS: ubuntu-latest - TARGET: aarch64-unknown-linux-gnu # tested on aws t4g.nano OS: ubuntu-latest - - TARGET: aarch64-unknown-linux-musl # tested on aws t4g.nano in alpine container - OS: ubuntu-latest + # - TARGET: aarch64-unknown-linux-musl # tested on aws t4g.nano in alpine container + # OS: ubuntu-latest - TARGET: armv7-unknown-linux-gnueabihf # raspberry pi 2-3-4, not tested OS: ubuntu-latest - - TARGET: armv7-unknown-linux-musleabihf # raspberry pi 2-3-4, not tested - OS: ubuntu-latest + # - TARGET: armv7-unknown-linux-musleabihf # raspberry pi 2-3-4, not tested + # OS: ubuntu-latest - TARGET: arm-unknown-linux-gnueabihf # raspberry pi 0-1, not tested OS: ubuntu-latest - - TARGET: arm-unknown-linux-musleabihf # raspberry pi 0-1, not tested - OS: ubuntu-latest + # - TARGET: arm-unknown-linux-musleabihf # raspberry pi 0-1, not tested + # OS: ubuntu-latest - TARGET: x86_64-apple-darwin # tested on a mac, is not properly signed so there are security warnings OS: macos-latest - TARGET: x86_64-pc-windows-msvc # tested on a windows machine @@ -77,7 +77,7 @@ jobs: # dependencies are only needed on ubuntu as that's the only place where # we make cross-compilation if [[ $OS =~ ^ubuntu.*$ ]]; then - sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf musl + sudo apt-get install -qq crossbuild-essential-arm64 crossbuild-essential-armhf musl musl-tools musl-dev fi # some additional configuration for cross-compilation on linux From 3f262db2d7234d47ba29c64f4e4f028c7e282830 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 19:54:17 +0100 Subject: [PATCH 28/38] perf: remove buffer allocations per-block --- src/database.rs | 19 ++++++++-------- src/main.rs | 21 ++++++++++++------ src/restoring.rs | 56 +++++++++++++++++++++++++++++++++++++----------- 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/database.rs b/src/database.rs index c69f1c2..5201831 100644 --- a/src/database.rs +++ b/src/database.rs @@ -247,23 +247,24 @@ impl DFileDatabase { self.inner.lock().unwrap().get_zip_by_block_id(block_id) } - pub fn get_content_block(&self, block_id: &BlockIdHash) -> Result>> { - let mut output = Vec::new(); - - //let mut zip = zip::ZipArchive::new(File::open(filename).unwrap()).unwrap(); + pub fn get_content_block( + &self, + block_id: &BlockIdHash, + block_buf: &mut Vec, + ) -> Result> { let ziparch = self.get_zip_by_block_id(block_id); if let Some(mut ziparch) = ziparch { - let buf = &mut [0u8; 48]; - let name_reencoded = block_id.as_base64_urlsafe(buf); + let base64_buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(base64_buf); let mut block = ziparch .by_name(name_reencoded) .wrap_err("block file by name not found even though we indexed it before")?; - block - .read_to_end(&mut output) + let n = block + .read_to_end(block_buf) .wrap_err_with(|| format!("reading block file {:?}", block_id))?; - Ok(Some(output)) + Ok(Some(n)) } else { Ok(None) } diff --git a/src/main.rs b/src/main.rs index d5e85f9..ba6a4b7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod sorting; mod stripbom; mod ziparchive; -use crate::restoring::{restore_file, RestoreParams, RestoreSummary}; +use crate::restoring::{restore_file, RestoreContext, RestoreParams, RestoreSummary}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; use blockid::*; @@ -244,7 +244,7 @@ fn restore_all( } else { "Verifying" }; - println!("{} directory structure", doing); + println!("{doing} directory structure"); let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new( params.summary.folder_count as u64, @@ -257,16 +257,19 @@ fn restore_all( .iter() .filter(|f| f.is_folder()) .par_bridge() - .try_for_each(|entry_folder| -> Result<()> { - restore_file(entry_folder, params).wrap_err("restoring dir")?; + .try_for_each_with(RestoreContext::new(), |ctx, entry_folder| -> Result<()> { + restore_file(entry_folder, params, ctx).wrap_err("restoring dir")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } Ok(()) })?; + if let Some(pb) = &pb { + pb.lock().unwrap().tick(); + } println!(); - println!("{} files", doing); + println!("{doing} files"); let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new( params.summary.predicted_bytes, @@ -278,13 +281,17 @@ fn restore_all( .iter() .filter(|f| f.is_file()) .par_bridge() - .try_for_each(|entry_file| -> Result<()> { - restore_file(entry_file, params).wrap_err("restoring file entry")?; + .try_for_each_with(RestoreContext::new(), |ctx, entry_file| -> Result<()> { + restore_file(entry_file, params, ctx).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().add(entry_file.predicted_time()); } Ok(()) })?; + if let Some(pb) = &pb { + pb.lock().unwrap().tick(); + } + println!(); Ok(()) } diff --git a/src/restoring.rs b/src/restoring.rs index f1b1080..84a29d3 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -14,7 +14,23 @@ use std::{ path::{Path, PathBuf}, }; +#[derive(Clone)] +pub struct RestoreContext { + pub block_buffer: RefCell>, + pub block_hashes_buffer: RefCell>, +} + +impl RestoreContext { + pub fn new() -> Self { + Self { + block_buffer: RefCell::new(Vec::with_capacity(8 * 1024)), + block_hashes_buffer: RefCell::new(Vec::with_capacity(8 * 1024)), + } + } +} + struct RestoreFileContext<'a> { + restore_context: &'a RestoreContext, db: &'a DFileDatabase, entry: &'a FileEntry, @@ -45,7 +61,11 @@ pub struct RestoreParams<'a> { pub summary: RestoreSummary, } -pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> { +pub fn restore_file( + entry: &FileEntry, + params: &RestoreParams<'_>, + restore_context: &RestoreContext, +) -> Result<()> { let (path, relative_file_path) = if let Some(restore_path) = ¶ms.restore_path { let root_path = Path::new(restore_path); let dfile_path = &entry.path[0..]; @@ -70,6 +90,7 @@ pub fn restore_file(entry: &FileEntry, params: &RestoreParams<'_>) -> Result<()> FileType::File { hash, size, .. } => { let hasher = if *size > 0 { Some(Sha256::new()) } else { None }; let context = RestoreFileContext { + restore_context, entry, db: params.db, debug_location: false, @@ -110,17 +131,19 @@ fn restore_file_singleblock(ctx: &RestoreFileContext<'_>) -> Result<()> { } else { None }; - let block = ctx.db.get_content_block(ctx.hash)?; - if let Some(block) = block { + let buf = &mut ctx.restore_context.block_buffer.borrow_mut(); + buf.clear(); + let block = ctx.db.get_content_block(ctx.hash, buf)?; + if let Some(_block_len) = block { if let Some(out_file) = &mut out_file { out_file - .write_all(block.as_ref()) + .write_all(buf.as_slice()) .wrap_err("write single-block file")?; } { let mut hasher = ctx.hasher.borrow_mut(); if let Some(h) = hasher.as_mut() { - h.update(block.as_slice()); + h.update(buf.as_slice()); } } } else if ctx.size > 0 { @@ -154,21 +177,26 @@ fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { // Each blockid points to a list of blockids for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { let blockhashoffset = blhi * ctx.db.offset_size(); + + let hashes_buf = &mut ctx.restore_context.block_hashes_buffer.borrow_mut(); + hashes_buf.clear(); let binary_hashes = ctx .db - .get_content_block(blh) + .get_content_block(blh, hashes_buf) .wrap_err_with(|| format!("get main content block: {}", blh))?; - if let Some(binary_hashes) = binary_hashes { + if let Some(_binary_hashes_len) = binary_hashes { let mut last_block_size = None; - for (bi, bhash) in binary_hashes.chunks(ctx.db.hash_size()).enumerate() { + for (bi, bhash) in hashes_buf.chunks(ctx.db.hash_size()).enumerate() { //let bhash = base64::encode(bhash); let bhash = BlockIdHash::from_bytes(bhash) .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; - let block = ctx.db.get_content_block(&bhash).wrap_err_with(|| { + let buf = &mut ctx.restore_context.block_buffer.borrow_mut(); + buf.clear(); + let block = ctx.db.get_content_block(&bhash, buf).wrap_err_with(|| { format!("get one of content blocks (number {}): {}", bi, blh) })?; - if let Some(block) = block { + if let Some(_block_len) = block { let full_block = ctx.db.block_size(); if let Some(out_file) = &mut out_file { @@ -176,12 +204,14 @@ fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { out_file .seek(SeekFrom::Start(offset)) .wrap_err("seek blockhashoffset + bi * full_block")?; - out_file.write_all(&block).wrap_err("write (multi) block")?; + out_file + .write_all(buf.as_slice()) + .wrap_err("write (multi) block")?; } { let mut hasher = ctx.hasher.borrow_mut(); if let Some(h) = hasher.as_mut() { - h.update(block.as_slice()); + h.update(buf.as_slice()); } } if ctx.strict_block_size { @@ -194,7 +224,7 @@ fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { ))?; } } - last_block_size = Some(block.len()); + last_block_size = Some(buf.len()); } } else { Err(eyre!( From a03d43c7772d97bb5c3671a59f50ed773952da41 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 20:02:19 +0100 Subject: [PATCH 29/38] dhat fix --- src/main.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index ba6a4b7..344dfe6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,13 +59,17 @@ struct CliArgs { fn main() { #[cfg(feature = "dhat-heap")] - std::thread::spawn(|| { - let _profiler = dhat::Profiler::new_heap(); + { + std::thread::spawn(|| { + let _profiler = dhat::Profiler::new_heap(); - std::thread::sleep(std::time::Duration::from_secs(4 * 60)); - }); + std::thread::sleep(std::time::Duration::from_secs(10 * 60)); + // save profile after 10 minutes + }); + + std::thread::sleep(std::time::Duration::from_millis(200)); + } - std::thread::sleep(std::time::Duration::from_millis(100)); let result = run(); match result { Err(err) => { From eb7b83b9aa36539433f30e3e855794defcf56d07 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 22:02:36 +0100 Subject: [PATCH 30/38] bump version --- Cargo.toml | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 17fb96e..a390ece 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "rust-duplicati-restore" description = "A program to restore duplicati backups" license = "MIT" -version = "0.0.3" +version = "0.0.4" authors = ["Nathan McCarty ", "7ERr0r"] edition = "2021" diff --git a/src/main.rs b/src/main.rs index 344dfe6..591af87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -218,7 +218,7 @@ fn run() -> Result<()> { "{} bytes on drive to be restored (predicted)", summary.predicted_bytes ); - + println!("Waiting for dblocks"); let dblock_db = db_join.join().unwrap()?; println!("Sorting file_entries"); From 92ca7bb129eb3dab8f2e311f0beabe9ca0427387 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Thu, 12 Jan 2023 22:33:03 +0100 Subject: [PATCH 31/38] add workflow permission write --- .github/workflows/rust.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fb7dfae..e942013 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -129,6 +129,9 @@ jobs: if: startsWith(github.ref, 'refs/tags/') needs: build runs-on: ubuntu-latest + permissions: + contents: write + packages: write steps: - name: Download artifacts uses: actions/download-artifact@v2 From 7cd16a7b55093a27e2fa6944ce6cf6d55ae5c368 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:26:41 +0100 Subject: [PATCH 32/38] update dependencies --- Cargo.toml | 6 +++--- src/blockhash.rs | 22 ++++++++++++++-------- src/database.rs | 13 ++++++------- src/ziparchive.rs | 24 ++++++++++++++++++++---- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a390ece..9770402 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "rust-duplicati-restore" description = "A program to restore duplicati backups" license = "MIT" -version = "0.0.4" +version = "0.0.5" authors = ["Nathan McCarty ", "7ERr0r"] edition = "2021" @@ -16,14 +16,14 @@ unqlite = ["dep:unqlite"] [dependencies] zip = { version = "*", git = "https://github.com/7ERr0r/zip-duplicati", rev = "77f115763e7d1e686273589e7b26f4efd3f5bf38" } chrono = "0.4.0" -base64 = "0.10.1" +base64 = "0.21" pbr = "1.0.1" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0.39" rayon = "1.0" #num_cpus = "1.10.0" unqlite = { version = "1.5", optional = true } -indicatif = "0.11.0" +indicatif = "0.17" clap = { version = "4.0.32", features = ["derive"] } eyre = "0.6.8" serde_path_to_error = "0.1" diff --git a/src/blockhash.rs b/src/blockhash.rs index bc6b5cd..acf0590 100644 --- a/src/blockhash.rs +++ b/src/blockhash.rs @@ -1,5 +1,6 @@ use std::{cell::RefCell, fmt::Display}; +use base64::engine::general_purpose; use smallvec::SmallVec; use crate::hexdisplay::HexDisplayBytes; @@ -30,18 +31,21 @@ impl BlockIdHash { } pub fn from_base64(block_id_str: &str) -> Option { - Self::from_base64_config(block_id_str, base64::STANDARD) + Self::from_base64_config(block_id_str, general_purpose::STANDARD) } #[allow(unused)] pub fn from_base64_urlsafe(block_id_str: &str) -> Option { - Self::from_base64_config(block_id_str, base64::URL_SAFE) + Self::from_base64_config(block_id_str, general_purpose::URL_SAFE) } - pub fn from_base64_config(block_id_str: &str, config: base64::Config) -> Option { + pub fn from_base64_config( + block_id_str: &str, + engine: E, + ) -> Option { BASE64_DECODE_BUF.with(|b| -> Option { let buffer: &mut Vec = &mut b.borrow_mut(); assert!(block_id_str.len() < buffer.capacity()); - base64::decode_config_buf(block_id_str, config, buffer).ok()?; + engine.decode_slice(block_id_str, buffer).ok()?; let hash = BlockIdHash { hash: SmallVec::from_slice(buffer), }; @@ -52,13 +56,15 @@ impl BlockIdHash { #[allow(unused)] pub fn as_base64<'a>(&self, buf: &'a mut [u8]) -> &'a str { - self.as_base64_config(base64::STANDARD, buf) + self.as_base64_config(general_purpose::STANDARD, buf) } pub fn as_base64_urlsafe<'a>(&self, buf: &'a mut [u8]) -> &'a str { - self.as_base64_config(base64::URL_SAFE, buf) + self.as_base64_config(general_purpose::URL_SAFE, buf) } - pub fn as_base64_config<'a>(&self, config: base64::Config, buf: &'a mut [u8]) -> &'a str { - let encoded_len = base64::encode_config_slice(self.hash.as_slice(), config, &mut buf[..]); + pub fn as_base64_config<'a, E: base64::Engine>(&self, engine: E, buf: &'a mut [u8]) -> &'a str { + let encoded_len = engine + .encode_slice(self.hash.as_slice(), &mut buf[..]) + .unwrap(); //debug_assert_eq!(encoded_len, buf.len()); std::str::from_utf8(&buf[..encoded_len]).expect("Invalid UTF8") diff --git a/src/database.rs b/src/database.rs index 5201831..4aefe88 100644 --- a/src/database.rs +++ b/src/database.rs @@ -4,6 +4,8 @@ use crate::ziparchive::MyCloneFileConfig; use crate::ziparchive::MyCloneFileReader; use crate::ziparchive::ZipArchiveWrapper; use crate::ziparchive::ZipLocation; +use base64::engine::general_purpose; +use base64::Engine; use eyre::Context; use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; @@ -160,7 +162,7 @@ impl DFileDatabase { ProgressStyle::default_bar() .template( "[{elapsed_precise}] {wide_bar:40.cyan/blue} {pos:>7}/{len:7} {msg} [{eta_precise}]", - ) + )? .progress_chars("##-"), ); paths.par_iter().try_for_each(|zip_path| { @@ -181,17 +183,14 @@ impl DFileDatabase { let zipbuf = MyCloneFileReader::new(config.clone())?; let zip = zip::ZipArchive::new(zipbuf)?; - let arc_zippath = Arc::new(ZipLocation { - path: zip_path.clone(), - path_str: zip_path.to_string_lossy().to_string(), - }); + let arc_zippath = Arc::new(ZipLocation { path: zip_path }); // Convert to a list of paths for (index, file_name) in zip.file_names_ordered().enumerate() { //let file_in_zip = zip.by_index_raw(file_index)?; //let file_name = file_in_zip.name().to_string(); let hash_path = file_name; - let hash = base64::decode_config(&hash_path, base64::URL_SAFE)?; + let hash = general_purpose::URL_SAFE.decode(hash_path)?; { if hash.len() > 32 { println!("warn: hash len:{} requires heap alloc", hash.len()); @@ -217,7 +216,7 @@ impl DFileDatabase { zip_path: arc_zippath.clone(), archive: zip, }; - let path_str = arc_zippath.path_str.clone(); + let path_str = arc_zippath.path.to_string_lossy().to_string(); inner.zip2ziparchive.insert(path_str, wrapper); } diff --git a/src/ziparchive.rs b/src/ziparchive.rs index 02e05fa..c7b208a 100644 --- a/src/ziparchive.rs +++ b/src/ziparchive.rs @@ -10,17 +10,33 @@ use zip::ZipArchive; #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] /// Path to dblock.zip pub struct ZipLocation { - pub path_str: String, + // pub path_str: String, pub path: PathBuf, } -#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct BlockLocation { + /// Which dblock.zip file + pub zip_path: Arc, + /// Which file inside the zip pub file_index: u32, +} - /// Which dblock.zip file - pub zip_path: Arc, +impl Ord for BlockLocation { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // First zip_path (which dblock.zip it is) + // then file_index inside the ZIP file + self.zip_path + .cmp(&other.zip_path) + .then_with(|| self.file_index.cmp(&other.file_index)) + } +} + +impl PartialOrd for BlockLocation { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } } pub struct ZipArchiveWrapper { From c9549f75b972f43993709db9be549ec86503affd Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:43:24 +0100 Subject: [PATCH 33/38] fix: decode_vec len --- src/blockhash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockhash.rs b/src/blockhash.rs index acf0590..dc78c30 100644 --- a/src/blockhash.rs +++ b/src/blockhash.rs @@ -45,7 +45,7 @@ impl BlockIdHash { BASE64_DECODE_BUF.with(|b| -> Option { let buffer: &mut Vec = &mut b.borrow_mut(); assert!(block_id_str.len() < buffer.capacity()); - engine.decode_slice(block_id_str, buffer).ok()?; + engine.decode_vec(block_id_str, buffer).ok()?; let hash = BlockIdHash { hash: SmallVec::from_slice(buffer), }; From 61d534905bbe4e4d5400aeed45da85f8a695162b Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Sun, 15 Jan 2023 00:59:17 +0100 Subject: [PATCH 34/38] refactor: more readable code with less blocks --- src/database.rs | 155 ++++++++++--------- src/main.rs | 6 +- src/restoring.rs | 382 +++++++++++++++++++++++++++-------------------- 3 files changed, 315 insertions(+), 228 deletions(-) diff --git a/src/database.rs b/src/database.rs index 4aefe88..adf17e7 100644 --- a/src/database.rs +++ b/src/database.rs @@ -69,6 +69,9 @@ pub struct HashToBlocks { /// Maps zip file name to a singleton zip reader zip2ziparchive: HashMap, + /// zip_entry_name -> zip_name + /// + /// takes a lot of RAM so it's not used by default hash2path: Option, } @@ -86,30 +89,33 @@ impl HashToBlocks { } } - // pub fn get_zip_path_by_base64(&self, block_id: &str) -> Option { - // if let Some(hash2path) = &self.hash2path { - // hash2path.get_zip_path_by_base64(block_id) - // } else { - // None - // } - // } - pub fn get_location_by_block_id(&self, block_id: &BlockIdHash) -> Option { if let Some(hash2path) = &self.hash2path { hash2path.get_location_by_block_id(block_id) } else { - let buf = &mut [0u8; 48]; - let name_reencoded = block_id.as_base64_urlsafe(buf); - for ziparch in self.zip2ziparchive.values() { - if let Some(index) = ziparch.archive.get_file_index(name_reencoded) { - return Some(BlockLocation { + self.get_location_by_block_id_purezip(block_id) + } + } + pub fn get_location_by_block_id_purezip( + &self, + block_id: &BlockIdHash, + ) -> Option { + let buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(buf); + for ziparch in self.zip2ziparchive.values() { + let location = + ziparch + .archive + .get_file_index(name_reencoded) + .map(|index| BlockLocation { file_index: index as u32, zip_path: ziparch.zip_path.clone(), }); - } + if location.is_some() { + return location; } - None } + None } pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { @@ -127,15 +133,22 @@ impl HashToBlocks { let zname = zname.map(|n| n.to_string_lossy().to_string()); zname.and_then(|zname| self.get_zip_archive(&zname)) } else { - let buf = &mut [0u8; 48]; - let name_reencoded = block_id.as_base64_urlsafe(buf); - for ziparch in self.zip2ziparchive.values() { - if ziparch.archive.contains_file_name(name_reencoded) { - return Some(ziparch.archive.clone()); - } + self.get_zip_by_block_id_purezip(block_id) + } + } + + pub fn get_zip_by_block_id_purezip( + &self, + block_id: &BlockIdHash, + ) -> Option> { + let buf = &mut [0u8; 48]; + let name_reencoded = block_id.as_base64_urlsafe(buf); + for ziparch in self.zip2ziparchive.values() { + if ziparch.archive.contains_file_name(name_reencoded) { + return Some(ziparch.archive.clone()); } - None } + None } } @@ -181,48 +194,65 @@ impl DFileDatabase { buf_capacity: AtomicU32::new(1024), }); let zipbuf = MyCloneFileReader::new(config.clone())?; - let zip = zip::ZipArchive::new(zipbuf)?; - - let arc_zippath = Arc::new(ZipLocation { path: zip_path }); - // Convert to a list of paths - - for (index, file_name) in zip.file_names_ordered().enumerate() { - //let file_in_zip = zip.by_index_raw(file_index)?; - //let file_name = file_in_zip.name().to_string(); - let hash_path = file_name; - let hash = general_purpose::URL_SAFE.decode(hash_path)?; - { - if hash.len() > 32 { - println!("warn: hash len:{} requires heap alloc", hash.len()); - } - let mut inner = self.inner.lock().unwrap(); - if let Some(hash2path) = &mut inner.hash2path { - hash2path.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_zippath.clone(), - file_index: index as u32, - }, - ); - } - } - } + let ziparch = zip::ZipArchive::new(zipbuf)?; + + let arc_ziploc = Arc::new(ZipLocation { path: zip_path }); - { - use std::sync::atomic::Ordering; - config.buf_capacity.store(32 * 1024, Ordering::Relaxed); - let mut inner = self.inner.lock().unwrap(); - let wrapper = ZipArchiveWrapper { - zip_path: arc_zippath.clone(), - archive: zip, - }; - let path_str = arc_zippath.path.to_string_lossy().to_string(); - inner.zip2ziparchive.insert(path_str, wrapper); + let mut inner = self.inner.lock().unwrap(); + + if let Some(hash2path) = &mut inner.hash2path { + self.register_hash_to_path(hash2path, &ziparch, arc_ziploc.clone())?; } + self.register_zip_archive(config, arc_ziploc, ziparch); + + Ok(()) + } + /// Remembers zip file names in a hashmap + /// + /// zip_entry_name -> zip_name + pub fn register_hash_to_path( + &self, + hash2path: &mut HashToPath, + ziparch: &ZipArchive, + arc_ziploc: Arc, + ) -> Result<()> { + for (index, file_name) in ziparch.file_names_ordered().enumerate() { + // file_name is a hash in base64 + let hash = general_purpose::URL_SAFE.decode(file_name)?; + + if hash.len() > 32 { + println!("warn: hash len:{} requires heap alloc", hash.len()); + } + + hash2path.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_ziploc.clone(), + file_index: index as u32, + }, + ); + } Ok(()) } + pub fn register_zip_archive( + &self, + config: Arc, + arc_ziploc: Arc, + ziparch: ZipArchive, + ) { + use std::sync::atomic::Ordering; + config.buf_capacity.store(32 * 1024, Ordering::Relaxed); + let mut inner = self.inner.lock().unwrap(); + let wrapper = ZipArchiveWrapper { + zip_path: arc_ziploc.clone(), + archive: ziparch, + }; + let path_str = arc_ziploc.path.to_string_lossy().to_string(); + inner.zip2ziparchive.insert(path_str, wrapper); + } + pub fn get_block_id_location(&self, block_id: &BlockIdHash) -> Option { self.inner .lock() @@ -230,15 +260,6 @@ impl DFileDatabase { .get_location_by_block_id(block_id) } - // pub fn get_zip_path_from_block_id(&self, block_id: &str) -> Option { - // self.inner.lock().unwrap().get_zip_path_by_base64(block_id) - // } - - // pub fn get_zip_archive(&self, zip_filename: &str) -> Option> { - // self.inner.lock().unwrap().get_zip_archive(zip_filename) - - // } - pub fn get_zip_by_block_id( &self, block_id: &BlockIdHash, diff --git a/src/main.rs b/src/main.rs index 591af87..ac4ef06 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod sorting; mod stripbom; mod ziparchive; -use crate::restoring::{restore_file, RestoreContext, RestoreParams, RestoreSummary}; +use crate::restoring::{restore_entry, RestoreContext, RestoreParams, RestoreSummary}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; use blockid::*; @@ -262,7 +262,7 @@ fn restore_all( .filter(|f| f.is_folder()) .par_bridge() .try_for_each_with(RestoreContext::new(), |ctx, entry_folder| -> Result<()> { - restore_file(entry_folder, params, ctx).wrap_err("restoring dir")?; + restore_entry(entry_folder, params, ctx).wrap_err("restoring dir")?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } @@ -286,7 +286,7 @@ fn restore_all( .filter(|f| f.is_file()) .par_bridge() .try_for_each_with(RestoreContext::new(), |ctx, entry_file| -> Result<()> { - restore_file(entry_file, params, ctx).wrap_err("restoring file entry")?; + restore_entry(entry_file, params, ctx).wrap_err("restoring file entry")?; if let Some(pb) = &pb { pb.lock().unwrap().add(entry_file.predicted_time()); } diff --git a/src/restoring.rs b/src/restoring.rs index 84a29d3..c82891e 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -45,6 +45,8 @@ struct RestoreFileContext<'a> { absolute_path: Option<&'a PathBuf>, /// None if only verifying relative_file_path: Option<&'a PathBuf>, + + out_file: RefCell>, } pub struct RestoreSummary { @@ -60,13 +62,9 @@ pub struct RestoreParams<'a> { pub replace_backslash_to_slash: bool, pub summary: RestoreSummary, } - -pub fn restore_file( - entry: &FileEntry, - params: &RestoreParams<'_>, - restore_context: &RestoreContext, -) -> Result<()> { - let (path, relative_file_path) = if let Some(restore_path) = ¶ms.restore_path { +/// Returns Some(absolute, relative) +pub fn calculate_path(entry: &FileEntry, params: &RestoreParams<'_>) -> Option<(PathBuf, PathBuf)> { + if let Some(restore_path) = ¶ms.restore_path { let root_path = Path::new(restore_path); let dfile_path = &entry.path[0..]; let mut dfile_path = dfile_path.replacen(":\\", "\\", 1); @@ -76,172 +74,237 @@ pub fn restore_file( let relative_file_path = PathBuf::from(&dfile_path); let path = Path::join(root_path, &relative_file_path); - (Some(path), Some(relative_file_path)) + Some((path, relative_file_path)) } else { - (None, None) - }; + None + } +} + +pub fn restore_entry( + entry: &FileEntry, + params: &RestoreParams<'_>, + restore_context: &RestoreContext, +) -> Result<()> { + let paths = calculate_path(entry, params); + let absolute_path = paths.as_ref().map(|v| &v.0); + let relative_file_path = paths.as_ref().map(|v| &v.1); match &entry.file_type { FileType::Folder { .. } => { - if let Some(path) = path { + if let Some(path) = absolute_path { fs::create_dir_all(path)?; } } FileType::File { hash, size, .. } => { - let hasher = if *size > 0 { Some(Sha256::new()) } else { None }; - let context = RestoreFileContext { + restore_file( + params, restore_context, - entry, - db: params.db, - debug_location: false, - strict_block_size: true, hash, - size: *size, - hasher: RefCell::new(hasher), - absolute_path: path.as_ref(), - relative_file_path: relative_file_path.as_ref(), - }; - - // Small files only have one block - if entry.block_lists.is_empty() { - restore_file_singleblock(&context)?; - } else { - restore_file_multiblock(&context)?; - } - - check_file_hash(&context)?; + *size, + absolute_path, + relative_file_path, + entry, + )?; } _ => (), } Ok(()) } - -fn restore_file_singleblock(ctx: &RestoreFileContext<'_>) -> Result<()> { - if ctx.debug_location { - let loc = ctx.db.get_block_id_location(ctx.hash); - println!( - "restoring file (single) {:?}, index:{:?}", - ctx.relative_file_path, - loc.map(|loc| loc.file_index) - ); - } - - let mut out_file = if let Some(path) = ctx.absolute_path { +fn restore_file( + params: &RestoreParams<'_>, + restore_context: &RestoreContext, + hash: &BlockIdHash, + size: i64, + absolute_path: Option<&PathBuf>, + relative_file_path: Option<&PathBuf>, + entry: &FileEntry, +) -> Result<()> { + let hasher = if size > 0 { Some(Sha256::new()) } else { None }; + let out_file = if let Some(path) = &absolute_path { Some(File::create(path)?) } else { None }; + let context = RestoreFileContext { + restore_context, + entry, + db: params.db, + debug_location: false, + strict_block_size: true, + hash, + size, + hasher: RefCell::new(hasher), + absolute_path, + relative_file_path, + out_file: RefCell::new(out_file), + }; + + // Small files only have one block + if entry.block_lists.is_empty() { + restore_file_singleblock(&context)?; + } else { + restore_file_multiblock(&context)?; + } + + check_file_hash(&context)?; + Ok(()) +} + +fn restore_file_singleblock(ctx: &RestoreFileContext<'_>) -> Result<()> { + debug_block_restore_maybe(ctx, true); + + if ctx.size <= 0 { + return Ok(()); + } + let buf = &mut ctx.restore_context.block_buffer.borrow_mut(); buf.clear(); let block = ctx.db.get_content_block(ctx.hash, buf)?; - if let Some(_block_len) = block { - if let Some(out_file) = &mut out_file { - out_file - .write_all(buf.as_slice()) - .wrap_err("write single-block file")?; - } - { - let mut hasher = ctx.hasher.borrow_mut(); - if let Some(h) = hasher.as_mut() { - h.update(buf.as_slice()); - } - } - } else if ctx.size > 0 { - Err(eyre!( - "Missing block {} for {:?}", - ctx.hash, - ctx.absolute_path - ))?; + let _len = block.ok_or(|| eyre!("Missing block {} for {:?}", ctx.hash, ctx.absolute_path)); + + if let Some(out_file) = ctx.out_file.borrow_mut().as_mut() { + out_file + .write_all(buf.as_slice()) + .wrap_err("write single-block file")?; } + update_hasher_maybe(ctx, buf); + Ok(()) } - -fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { - if ctx.debug_location { - let loc = ctx - .entry - .block_lists - .first() - .and_then(|hash| ctx.db.get_block_id_location(hash)); - println!( - "restoring file (blocks) {:?}, index:{:?}", - ctx.relative_file_path, - loc.map(|loc| loc.file_index) - ); +fn debug_block_restore_maybe(ctx: &RestoreFileContext<'_>, is_multi: bool) { + if !ctx.debug_location { + return; } - let mut out_file = if let Some(path) = ctx.absolute_path { - Some(File::create(path)?) + + let multi_or_single = if is_multi { "multi" } else { "single" }; + let hash = if is_multi { + Some(ctx.hash) } else { - None + ctx.entry.block_lists.first() }; + let loc = hash.and_then(|hash| ctx.db.get_block_id_location(hash)); + println!( + "restoring file ({}) {:?}, index:{:?}", + multi_or_single, + ctx.relative_file_path, + loc.map(|loc| loc.file_index) + ); +} + +fn restore_file_multiblock(ctx: &RestoreFileContext<'_>) -> Result<()> { + debug_block_restore_maybe(ctx, true); + // Each blockid points to a list of blockids - for (blhi, blh) in ctx.entry.block_lists.iter().enumerate() { - let blockhashoffset = blhi * ctx.db.offset_size(); + for (main_hash_index, main_hash) in ctx.entry.block_lists.iter().enumerate() { + restore_file_multiblock_main(ctx, main_hash_index, main_hash)?; + } - let hashes_buf = &mut ctx.restore_context.block_hashes_buffer.borrow_mut(); - hashes_buf.clear(); - let binary_hashes = ctx - .db - .get_content_block(blh, hashes_buf) - .wrap_err_with(|| format!("get main content block: {}", blh))?; - if let Some(_binary_hashes_len) = binary_hashes { - let mut last_block_size = None; - for (bi, bhash) in hashes_buf.chunks(ctx.db.hash_size()).enumerate() { - //let bhash = base64::encode(bhash); - let bhash = BlockIdHash::from_bytes(bhash) - .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; - let buf = &mut ctx.restore_context.block_buffer.borrow_mut(); - buf.clear(); - let block = ctx.db.get_content_block(&bhash, buf).wrap_err_with(|| { - format!("get one of content blocks (number {}): {}", bi, blh) - })?; - - if let Some(_block_len) = block { - let full_block = ctx.db.block_size(); - - if let Some(out_file) = &mut out_file { - let offset = (blockhashoffset + bi * full_block) as u64; - out_file - .seek(SeekFrom::Start(offset)) - .wrap_err("seek blockhashoffset + bi * full_block")?; - out_file - .write_all(buf.as_slice()) - .wrap_err("write (multi) block")?; - } - { - let mut hasher = ctx.hasher.borrow_mut(); - if let Some(h) = hasher.as_mut() { - h.update(buf.as_slice()); - } - } - if ctx.strict_block_size { - if let Some(last) = last_block_size { - if last != full_block { - Err(eyre!( - "last block size != full_block, {} != {}", - last, - full_block - ))?; - } - } - last_block_size = Some(buf.len()); - } - } else { - Err(eyre!( - "Failed to find block {} for {:?}", - bhash, - ctx.absolute_path - ))?; - } - } - } else { + Ok(()) +} + +fn update_hasher_maybe(ctx: &RestoreFileContext<'_>, buf: &[u8]) { + let mut hasher = ctx.hasher.borrow_mut(); + if let Some(h) = hasher.as_mut() { + h.update(buf); + } +} + +fn restore_file_multiblock_block( + ctx: &RestoreFileContext<'_>, + block_index: usize, + block_hash: &[u8], + blockhashoffset: usize, + last_block_size: &mut Option, +) -> Result<()> { + //let bhash = base64::encode(bhash); + let block_hash = BlockIdHash::from_bytes(block_hash) + .ok_or_else(|| eyre!("binary hash len is not 32 bytes"))?; + let buf = &mut ctx.restore_context.block_buffer.borrow_mut(); + buf.clear(); + let block = ctx + .db + .get_content_block(&block_hash, buf) + .wrap_err_with(|| { + format!( + "get one of content blocks (number {}): {}", + block_index, block_hash + ) + })?; + + let _block_len = block.ok_or_else(|| { + eyre!( + "Failed to find block {} for {:?}", + block_hash, + ctx.absolute_path + ) + })?; + + if let Some(out_file) = ctx.out_file.borrow_mut().as_mut() { + let full_block = ctx.db.block_size(); + let offset = (blockhashoffset + block_index * full_block) as u64; + out_file + .seek(SeekFrom::Start(offset)) + .wrap_err("seek blockhashoffset + bi * full_block")?; + out_file + .write_all(buf.as_slice()) + .wrap_err("write (multi) block")?; + } + update_hasher_maybe(ctx, buf); + check_strict_block(ctx, buf, last_block_size)?; + + Ok(()) +} + +fn check_strict_block( + ctx: &RestoreFileContext<'_>, + buf: &[u8], + last_block_size: &mut Option, +) -> Result<()> { + if !ctx.strict_block_size { + return Ok(()); + } + if let Some(last) = last_block_size { + let full_block = ctx.db.block_size(); + if *last != full_block { Err(eyre!( - "Failed to find blocklist {} for {:?}", - blh, - ctx.absolute_path, + "last block size != full_block, {} != {}", + last, + full_block ))?; } } + *last_block_size = Some(buf.len()); + + Ok(()) +} + +fn restore_file_multiblock_main( + ctx: &RestoreFileContext<'_>, + main_hash_index: usize, + main_hash: &BlockIdHash, +) -> Result<()> { + let blockhashoffset = main_hash_index * ctx.db.offset_size(); + + let hashes_buf: &mut Vec = &mut ctx.restore_context.block_hashes_buffer.borrow_mut(); + let binary_hashes_len = { + hashes_buf.clear(); + ctx.db + .get_content_block(main_hash, hashes_buf) + .wrap_err_with(|| format!("get main content block: {}", main_hash))? + }; + + let _len = binary_hashes_len.ok_or_else(|| { + eyre!( + "Failed to find blocklist {} for {:?}", + main_hash, + ctx.absolute_path, + ) + })?; + + let mut last_block_size = None; + for (bi, bhash) in hashes_buf.chunks(ctx.db.hash_size()).enumerate() { + restore_file_multiblock_block(ctx, bi, bhash, blockhashoffset, &mut last_block_size)?; + } Ok(()) } @@ -256,25 +319,28 @@ fn check_file_hash(ctx: &RestoreFileContext<'_>) -> Result<()> { hasher }; - if let Some(hasher) = hasher { - let calculated_hash: &[u8] = &hasher.finalize()[..]; - let expected_hash = ctx.hash.hash.as_slice(); - if expected_hash == calculated_hash { - let debug_hash = false; - if debug_hash { - println!( - "hash is valid {} == {}", - HexDisplayBytes(expected_hash), - HexDisplayBytes(calculated_hash) - ); - } - } else { - Err(eyre!( - "hash is invalid: expected != calculated, {} != {}", - HexDisplayBytes(expected_hash), - HexDisplayBytes(calculated_hash) - ))? - } + if hasher.is_none() { + return Ok(()); } + let hasher = hasher.unwrap(); + + let calculated_hash: &[u8] = &hasher.finalize()[..]; + let expected_hash = ctx.hash.hash.as_slice(); + if expected_hash != calculated_hash { + return Err(eyre!( + "hash is invalid: expected != calculated, {} != {}", + HexDisplayBytes(expected_hash), + HexDisplayBytes(calculated_hash) + )); + } + let debug_hash = false; + if debug_hash { + println!( + "hash is valid {} == {}", + HexDisplayBytes(expected_hash), + HexDisplayBytes(calculated_hash) + ); + } + Ok(()) } From e452d2f074cc7c27b370d867bfd27d806cabde83 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Sun, 15 Jan 2023 01:47:42 +0100 Subject: [PATCH 35/38] refactor sorting --- src/database.rs | 46 ++++++------ src/{blockid.rs => dfileentry.rs} | 45 +++--------- src/dfiletype.rs | 32 +++++++++ src/main.rs | 114 ++++++++++++++++++------------ src/restoring.rs | 9 ++- src/sorting.rs | 4 +- 6 files changed, 139 insertions(+), 111 deletions(-) rename src/{blockid.rs => dfileentry.rs} (81%) create mode 100644 src/dfiletype.rs diff --git a/src/database.rs b/src/database.rs index adf17e7..fd5a3f4 100644 --- a/src/database.rs +++ b/src/database.rs @@ -6,13 +6,13 @@ use crate::ziparchive::ZipArchiveWrapper; use crate::ziparchive::ZipLocation; use base64::engine::general_purpose; use base64::Engine; +use eyre::eyre; use eyre::Context; use eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::IntoParallelRefIterator; use rayon::prelude::ParallelIterator; use serde::Deserialize; - use smallvec::SmallVec; use std::collections::HashMap; use std::io::Read; @@ -178,9 +178,12 @@ impl DFileDatabase { )? .progress_chars("##-"), ); - paths.par_iter().try_for_each(|zip_path| { + paths.par_iter().try_for_each(|zip_path| -> Result<()> { self.import_from_zip(zip_path) - .wrap_err_with(|| format!("import_from_zip: {:?}", zip_path)) + .wrap_err_with(|| format!("import_from_zip: {:?}", zip_path))?; + pb.inc(1); + + Ok(()) })?; Ok(()) @@ -198,10 +201,8 @@ impl DFileDatabase { let arc_ziploc = Arc::new(ZipLocation { path: zip_path }); - let mut inner = self.inner.lock().unwrap(); - - if let Some(hash2path) = &mut inner.hash2path { - self.register_hash_to_path(hash2path, &ziparch, arc_ziploc.clone())?; + if self.inner.lock().unwrap().hash2path.is_some() { + self.register_hash_to_path(&ziparch, arc_ziploc.clone())?; } self.register_zip_archive(config, arc_ziploc, ziparch); @@ -213,7 +214,6 @@ impl DFileDatabase { /// zip_entry_name -> zip_name pub fn register_hash_to_path( &self, - hash2path: &mut HashToPath, ziparch: &ZipArchive, arc_ziploc: Arc, ) -> Result<()> { @@ -222,16 +222,19 @@ impl DFileDatabase { let hash = general_purpose::URL_SAFE.decode(file_name)?; if hash.len() > 32 { - println!("warn: hash len:{} requires heap alloc", hash.len()); + Err(eyre!("warn: hash len:{} requires heap alloc", hash.len()))? } - hash2path.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_ziploc.clone(), - file_index: index as u32, - }, - ); + let mut inner = self.inner.lock().unwrap(); + if let Some(hash2path) = &mut inner.hash2path { + hash2path.hash2path.insert( + hash.into(), + BlockLocation { + zip_path: arc_ziploc.clone(), + file_index: index as u32, + }, + ); + } } Ok(()) } @@ -244,13 +247,16 @@ impl DFileDatabase { ) { use std::sync::atomic::Ordering; config.buf_capacity.store(32 * 1024, Ordering::Relaxed); - let mut inner = self.inner.lock().unwrap(); + let path_str = arc_ziploc.path.to_string_lossy().to_string(); let wrapper = ZipArchiveWrapper { - zip_path: arc_ziploc.clone(), + zip_path: arc_ziploc, archive: ziparch, }; - let path_str = arc_ziploc.path.to_string_lossy().to_string(); - inner.zip2ziparchive.insert(path_str, wrapper); + + { + let mut inner = self.inner.lock().unwrap(); + inner.zip2ziparchive.insert(path_str, wrapper); + } } pub fn get_block_id_location(&self, block_id: &BlockIdHash) -> Option { diff --git a/src/blockid.rs b/src/dfileentry.rs similarity index 81% rename from src/blockid.rs rename to src/dfileentry.rs index 9f2020b..2bdbb76 100644 --- a/src/blockid.rs +++ b/src/dfileentry.rs @@ -1,5 +1,7 @@ use crate::blockhash::BlockIdHash; +use crate::dfiletype::FileType; use crate::stripbom::strip_bom_from_bufread; +use crate::FileEntries; use eyre::eyre; use eyre::Context; use eyre::Result; @@ -9,38 +11,7 @@ use serde_json::Deserializer; use smallvec::SmallVec; use std::io::prelude::*; -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] -pub enum FileType { - File { - hash: BlockIdHash, - size: i64, - time: String, - }, - Folder { - metablockhash: String, - }, - SymLink, -} - -impl FileType { - pub fn is_file(&self) -> bool { - matches!(self, FileType::File { .. }) - } - - #[allow(unused)] - pub fn is_nonzero_file(&self) -> bool { - match self { - FileType::File { size, .. } => *size > 0, - _ => false, - } - } - - pub fn is_folder(&self) -> bool { - matches!(self, FileType::Folder { .. }) - } -} - -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] pub struct FileEntry { pub path: String, #[allow(unused)] @@ -108,7 +79,7 @@ impl FileEntry { /// How much bytes it probably takes on disk when restoring pub fn predicted_time(&self) -> u64 { // Not an accurate number - let psize = 8 * 1024 + self.path.len() as u64; + let psize = 4 * 1024 + self.path.len() as u64; if let FileType::File { size, .. } = self.file_type { psize + size as u64 } else { @@ -144,7 +115,7 @@ pub(self) struct IEntry { #[allow(unused)] /// Accepts the dlist as a string (must be read in first) /// Returns a Vec of FileEntrys -pub fn parse_dlist(dlist: &[u8]) -> Result> { +pub fn parse_dlist(dlist: &[u8]) -> Result { let file_entries = parse_dlist_read(dlist)?; Ok(file_entries) @@ -152,7 +123,7 @@ pub fn parse_dlist(dlist: &[u8]) -> Result> { /// Accepts the dlist as a Read trait /// Returns a Vec of FileEntrys -pub fn parse_dlist_read(mut rdr: R) -> Result> { +pub fn parse_dlist_read(mut rdr: R) -> Result { let mut file_entries = Vec::new(); strip_bom_from_bufread(&mut rdr)?; @@ -167,5 +138,7 @@ pub fn parse_dlist_read(mut rdr: R) -> Result> { file_entries.push(entry); } - Ok(file_entries) + Ok(FileEntries { + entries: file_entries, + }) } diff --git a/src/dfiletype.rs b/src/dfiletype.rs new file mode 100644 index 0000000..10a0193 --- /dev/null +++ b/src/dfiletype.rs @@ -0,0 +1,32 @@ +use crate::blockhash::BlockIdHash; + +#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] +pub enum FileType { + File { + hash: BlockIdHash, + size: i64, + time: String, + }, + Folder { + metablockhash: String, + }, + SymLink, +} + +impl FileType { + pub fn is_file(&self) -> bool { + matches!(self, FileType::File { .. }) + } + + #[allow(unused)] + pub fn is_nonzero_file(&self) -> bool { + match self { + FileType::File { size, .. } => *size > 0, + _ => false, + } + } + + pub fn is_folder(&self) -> bool { + matches!(self, FileType::Folder { .. }) + } +} diff --git a/src/main.rs b/src/main.rs index ac4ef06..d870ab6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,9 @@ #![warn(rust_2018_idioms)] mod blockhash; -mod blockid; mod database; +mod dfileentry; +mod dfiletype; mod hexdisplay; mod restoring; mod sorting; @@ -12,9 +13,9 @@ mod ziparchive; use crate::restoring::{restore_entry, RestoreContext, RestoreParams, RestoreSummary}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; -use blockid::*; use clap::Parser; use database::*; +use dfileentry::*; use eyre::eyre; use eyre::{Context, Result}; use pbr::ProgressBar; @@ -96,8 +97,12 @@ fn path_is_dblock_zip>(path: P) -> bool { filename_ends_with(path, "dblock.zip") } +pub struct FileEntries { + pub entries: Vec, +} + /// Open dlist file and parse json inside -fn parse_dlist_file>(dlist_path: P) -> Result> { +fn parse_dlist_file>(dlist_path: P) -> Result { let dlist_reader = File::open(dlist_path.as_ref()) .wrap_err_with(|| format!("open {:?}", dlist_path.as_ref()))?; let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; @@ -191,49 +196,19 @@ fn run() -> Result<()> { println!("Parsing dlist"); let file_entries = parse_dlist_file(newest_dlist)?; - let file_entries = Arc::new(file_entries); - - let entries = file_entries.clone(); - let summary_join = std::thread::spawn(move || { - let file_count = entries.iter().filter(|f| f.is_file()).count(); - let folder_count = entries.iter().filter(|f| f.is_folder()).count(); - let predicted_bytes: u64 = entries.iter().map(|f| f.predicted_time()).sum(); - let total_bytes: u64 = entries.iter().map(|f| f.bytes_size()).sum(); - RestoreSummary { - file_count, - folder_count, - total_bytes, - predicted_bytes, - } - }); - - println!(); + let summary = calculate_summary(&file_entries.entries); - let summary = summary_join.join().unwrap(); - - println!("{} files to be restored", summary.file_count); - println!("{} folders to be restored", summary.folder_count); - println!("{} bytes in files", summary.total_bytes); - println!( - "{} bytes on drive to be restored (predicted)", - summary.predicted_bytes - ); - println!("Waiting for dblocks"); let dblock_db = db_join.join().unwrap()?; - println!("Sorting file_entries"); - - let mut file_entries = Arc::try_unwrap(file_entries).expect("no other owners of Arc"); - - sort_files_sequentially(&mut file_entries, &dblock_db); + print_summary(&summary); let restore_params = RestoreParams { - db: &dblock_db, + db: Arc::new(dblock_db), restore_path: restore_dir, replace_backslash_to_slash: args.replace_backslash_to_slash.unwrap_or(!cfg!(windows)), summary, }; - restore_all(&args, &restore_params, &file_entries)?; + restore_all(&args, &restore_params, file_entries)?; Ok(()) } @@ -241,14 +216,20 @@ fn run() -> Result<()> { fn restore_all( args: &CliArgs, params: &RestoreParams<'_>, - file_entries: &[FileEntry], + file_entries: FileEntries, ) -> Result<()> { + let folders: Vec = file_entries + .entries + .iter() + .filter(|f| f.is_folder()) + .cloned() + .collect(); + println!("Sorting file_entries"); let doing = if params.restore_path.is_some() { "Restoring" } else { "Verifying" }; - println!("{doing} directory structure"); let pb = if args.progress_bar { Some(Arc::new(Mutex::new(ProgressBar::new( params.summary.folder_count as u64, @@ -257,20 +238,35 @@ fn restore_all( None }; - file_entries - .iter() - .filter(|f| f.is_folder()) - .par_bridge() - .try_for_each_with(RestoreContext::new(), |ctx, entry_folder| -> Result<()> { - restore_entry(entry_folder, params, ctx).wrap_err("restoring dir")?; + let dbc = params.db.clone(); + let sort_join = std::thread::spawn(move || -> FileEntries { + let mut file_entries = file_entries; + sort_files_sequentially(&mut file_entries.entries, &dbc); + file_entries + }); + + println!("{doing} directory structure"); + + folders.iter().par_bridge().try_for_each_with( + RestoreContext::new(), + |ctx, entry_folder| -> Result<()> { + restore_entry(entry_folder, params, ctx) + .wrap_err_with(|| format!("restoring dir {:?}", entry_folder.path))?; if let Some(pb) = &pb { pb.lock().unwrap().inc(); } Ok(()) - })?; + }, + )?; if let Some(pb) = &pb { pb.lock().unwrap().tick(); } + + if !sort_join.is_finished() { + println!("Waiting for sorting to finish"); + } + let file_entries = sort_join.join().unwrap(); + println!(); println!("{doing} files"); @@ -282,11 +278,13 @@ fn restore_all( None }; file_entries + .entries .iter() .filter(|f| f.is_file()) .par_bridge() .try_for_each_with(RestoreContext::new(), |ctx, entry_file| -> Result<()> { - restore_entry(entry_file, params, ctx).wrap_err("restoring file entry")?; + restore_entry(entry_file, params, ctx) + .wrap_err_with(|| format!("restoring file {:?}", entry_file.path))?; if let Some(pb) = &pb { pb.lock().unwrap().add(entry_file.predicted_time()); } @@ -299,6 +297,28 @@ fn restore_all( Ok(()) } +fn calculate_summary(entries: &[FileEntry]) -> RestoreSummary { + let file_count = entries.iter().filter(|f| f.is_file()).count(); + let folder_count = entries.iter().filter(|f| f.is_folder()).count(); + let predicted_bytes: u64 = entries.iter().map(|f| f.predicted_time()).sum(); + let total_bytes: u64 = entries.iter().map(|f| f.bytes_size()).sum(); + RestoreSummary { + file_count, + folder_count, + total_bytes, + predicted_bytes, + } +} + +fn print_summary(summary: &RestoreSummary) { + println!("{} files to be restored", summary.file_count); + println!("{} folders to be restored", summary.folder_count); + println!("{} bytes in files", summary.total_bytes); + println!( + "{} bytes on drive to be restored (predicted)", + summary.predicted_bytes + ); +} #[cfg(feature = "dhat-heap")] #[global_allocator] diff --git a/src/restoring.rs b/src/restoring.rs index c82891e..c78dcac 100644 --- a/src/restoring.rs +++ b/src/restoring.rs @@ -1,12 +1,11 @@ use crate::{ - blockhash::BlockIdHash, - blockid::{FileEntry, FileType}, - database::DFileDatabase, + blockhash::BlockIdHash, database::DFileDatabase, dfileentry::FileEntry, dfiletype::FileType, hexdisplay::HexDisplayBytes, }; use eyre::eyre; use eyre::{Context, Result}; use sha2::{Digest, Sha256}; +use std::sync::Arc; use std::{ cell::RefCell, fs::{self, File}, @@ -57,7 +56,7 @@ pub struct RestoreSummary { } pub struct RestoreParams<'a> { - pub db: &'a DFileDatabase, + pub db: Arc, pub restore_path: Option<&'a str>, pub replace_backslash_to_slash: bool, pub summary: RestoreSummary, @@ -128,7 +127,7 @@ fn restore_file( let context = RestoreFileContext { restore_context, entry, - db: params.db, + db: ¶ms.db, debug_location: false, strict_block_size: true, hash, diff --git a/src/sorting.rs b/src/sorting.rs index 431b1e2..091c048 100644 --- a/src/sorting.rs +++ b/src/sorting.rs @@ -1,7 +1,5 @@ use crate::{ - blockid::{FileEntry, FileType}, - database::DFileDatabase, - ziparchive::BlockLocation, + database::DFileDatabase, dfileentry::FileEntry, dfiletype::FileType, ziparchive::BlockLocation, }; use std::cmp::Ordering; From 107b0ff8e703ca4db2e3e4dad5bb0622a833d37c Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Sun, 15 Jan 2023 01:48:33 +0100 Subject: [PATCH 36/38] v0.0.6 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9770402..bd38ffa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "rust-duplicati-restore" description = "A program to restore duplicati backups" license = "MIT" -version = "0.0.5" +version = "0.0.6" authors = ["Nathan McCarty ", "7ERr0r"] edition = "2021" From 6859c440a0bdf98308669a67546ca63b246c8964 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Tue, 17 Jan 2023 01:28:31 +0100 Subject: [PATCH 37/38] refactor for better readability --- Cargo.toml | 7 +++--- README.md | 36 ++++++++++------------------ src/database.rs | 60 +++++++++++++++++++++++------------------------ src/ziparchive.rs | 23 ++++++++++++++---- 4 files changed, 63 insertions(+), 63 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd38ffa..7e8c9bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,15 +10,15 @@ edition = "2021" opt-level = 2 [features] -dhat-heap = ["dep:dhat"] # if you are doing heap profiling -unqlite = ["dep:unqlite"] +dhat-heap = ["dep:dhat"] # if you are doing heap profiling +unqlite = ["dep:unqlite"] # TODO maybe for veeery large backups [dependencies] zip = { version = "*", git = "https://github.com/7ERr0r/zip-duplicati", rev = "77f115763e7d1e686273589e7b26f4efd3f5bf38" } chrono = "0.4.0" base64 = "0.21" pbr = "1.0.1" -serde = {version = "1.0", features = ["derive"]} +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.39" rayon = "1.0" #num_cpus = "1.10.0" @@ -31,4 +31,3 @@ smallvec = "*" sha2 = "0.10.6" crossbeam-channel = "0.5.6" dhat = { version = "0.3.2", optional = true } - diff --git a/README.md b/README.md index c1b67eb..e039df4 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,30 @@ # Rust Duplicati Restore -Rust program for duplicati disaster recovery quick, fast, and in a hurry. -Uses rayon to process files across many threads, to maximze restore speed. +[Fast](https://programming-language-benchmarks.vercel.app/rust-vs-csharp) [Duplicati](https://github.com/duplicati/duplicati) [disaster](https://duplicati.readthedocs.io/en/stable/08-disaster-recovery/) [recovery](https://github.com/duplicati/duplicati/tree/master/Duplicati/CommandLine/RecoveryTool). +Processes files across many threads, to maximze restore speed. -## Getting Started - -Simply run the rust-duplicati-restore from the commandline. +## Run ``` Usage: cargo run -- --backup-dir --restore-dir ``` -### Prerequisites - -You must have sqlite3 installed on your system for this program to function. - - -### Installing - -Simply run -``` -cargo build --release -``` +[More flags here](https://github.com/7ERr0r/duplicati-restore-rs/blob/master/src/main.rs#L31) -Or download the latest binary from the artifacts +Or download the latest [binary from releases](https://github.com/7ERr0r/duplicati-restore-rs/releases) ## Limitations -* Does not yet support encrypted backups, I reccomend combining aescrypt with gnu parallel for decryption -* Does not support remote repositories yet, I reccomend using rclone to pull donw a local copy +* Does not yet support [encrypted backups](https://github.com/duplicati/duplicati/issues/2927) - `.aes` files +* Does not support [remote repositories](https://crates.io/crates/remotefs) yet, I reccomend using rclone to pull down a local copy ## Built With -* [Rust](https://www.rust-lang.org/) -* [SQLite](https://www.sqlite.org) -* [Rayon](https://github.com/rayon-rs/rayon) -* And may more, see Cargo.toml for full list +* [Rust](https://www.rust-lang.org/) +* [`rayon` crate](https://github.com/rayon-rs/rayon) +* [Modified `zip` crate](https://github.com/7ERr0r/zip-duplicati) +* And many more, see [Cargo.toml](Cargo.toml) for full list ## License @@ -47,6 +35,6 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md * Ben Fisher - His python script included in the Duplicati reposistory inspired this project, and this project was roughly based on it. -* Nathan McCarty - Created Rust-Duplicati-Restore itself +* Nathan McCarty - Created [Rust-Duplicati-Restore](https://github.com/nmccarty/Rust-Duplicati-Restore) itself * 7ERr0r - Optimized ZIP reader. Added sha2 verification. diff --git a/src/database.rs b/src/database.rs index fd5a3f4..f672dfd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -58,12 +58,27 @@ impl HashToPath { pub fn get_zip_path_by_block_id(&self, block_id: &BlockIdHash) -> Option { self.hash2path .get(&block_id.hash) - .map(|v| v.zip_path.path.clone()) + .map(|v| v.ziplocation.path.clone()) } pub fn get_location_by_block_id(&self, block_id: &BlockIdHash) -> Option { self.hash2path.get(&block_id.hash).cloned() } + + pub fn insert_location( + &mut self, + hash: SmallVec<[u8; 32]>, + ziplocation: &Arc, + entry_index: usize, + ) { + self.hash2path.insert( + hash.into(), + BlockLocation { + ziplocation: ziplocation.clone(), + file_index: entry_index as u32, + }, + ); + } } pub struct HashToBlocks { /// Maps zip file name to a singleton zip reader @@ -77,15 +92,13 @@ pub struct HashToBlocks { impl HashToBlocks { pub fn new(use_hash_to_path: bool) -> Self { - let hash2path = if use_hash_to_path { - Some(HashToPath::new()) - } else { - None - }; - let zip2ziparchive = HashMap::new(); Self { - hash2path, - zip2ziparchive, + hash2path: if use_hash_to_path { + Some(HashToPath::new()) + } else { + None + }, + zip2ziparchive: HashMap::new(), } } @@ -103,14 +116,7 @@ impl HashToBlocks { let buf = &mut [0u8; 48]; let name_reencoded = block_id.as_base64_urlsafe(buf); for ziparch in self.zip2ziparchive.values() { - let location = - ziparch - .archive - .get_file_index(name_reencoded) - .map(|index| BlockLocation { - file_index: index as u32, - zip_path: ziparch.zip_path.clone(), - }); + let location = ziparch.get_block_location(name_reencoded); if location.is_some() { return location; } @@ -144,7 +150,7 @@ impl HashToBlocks { let buf = &mut [0u8; 48]; let name_reencoded = block_id.as_base64_urlsafe(buf); for ziparch in self.zip2ziparchive.values() { - if ziparch.archive.contains_file_name(name_reencoded) { + if ziparch.contains_file_name(name_reencoded) { return Some(ziparch.archive.clone()); } } @@ -159,8 +165,6 @@ pub struct DFileDatabase { impl DFileDatabase { pub fn new(manifest_bytes: &[u8], use_hash_to_path: bool) -> Result { - // let conn = UnQLite::create(file); - // conn.kv_store("test_key_name", "test_key_value").map_err(|_| eyre!("can't write to database"))?; let manifest: Manifest = serde_json::from_slice(manifest_bytes)?; let inner = Arc::new(Mutex::new(HashToBlocks::new(use_hash_to_path))); @@ -215,7 +219,7 @@ impl DFileDatabase { pub fn register_hash_to_path( &self, ziparch: &ZipArchive, - arc_ziploc: Arc, + ziplocation: Arc, ) -> Result<()> { for (index, file_name) in ziparch.file_names_ordered().enumerate() { // file_name is a hash in base64 @@ -227,13 +231,7 @@ impl DFileDatabase { let mut inner = self.inner.lock().unwrap(); if let Some(hash2path) = &mut inner.hash2path { - hash2path.hash2path.insert( - hash.into(), - BlockLocation { - zip_path: arc_ziploc.clone(), - file_index: index as u32, - }, - ); + hash2path.insert_location(hash.into(), &ziplocation, index); } } Ok(()) @@ -242,14 +240,14 @@ impl DFileDatabase { pub fn register_zip_archive( &self, config: Arc, - arc_ziploc: Arc, + ziplocation: Arc, ziparch: ZipArchive, ) { use std::sync::atomic::Ordering; config.buf_capacity.store(32 * 1024, Ordering::Relaxed); - let path_str = arc_ziploc.path.to_string_lossy().to_string(); + let path_str = ziplocation.path.to_string_lossy().to_string(); let wrapper = ZipArchiveWrapper { - zip_path: arc_ziploc, + ziplocation, archive: ziparch, }; diff --git a/src/ziparchive.rs b/src/ziparchive.rs index c7b208a..bb44931 100644 --- a/src/ziparchive.rs +++ b/src/ziparchive.rs @@ -17,7 +17,7 @@ pub struct ZipLocation { #[derive(Clone, Debug, PartialEq, Eq)] pub struct BlockLocation { /// Which dblock.zip file - pub zip_path: Arc, + pub ziplocation: Arc, /// Which file inside the zip pub file_index: u32, @@ -27,8 +27,8 @@ impl Ord for BlockLocation { fn cmp(&self, other: &Self) -> std::cmp::Ordering { // First zip_path (which dblock.zip it is) // then file_index inside the ZIP file - self.zip_path - .cmp(&other.zip_path) + self.ziplocation + .cmp(&other.ziplocation) .then_with(|| self.file_index.cmp(&other.file_index)) } } @@ -40,10 +40,25 @@ impl PartialOrd for BlockLocation { } pub struct ZipArchiveWrapper { - pub zip_path: Arc, + pub ziplocation: Arc, pub archive: ZipArchive, } +impl ZipArchiveWrapper { + pub fn get_block_location(&self, block_base64: &str) -> Option { + self.archive + .get_file_index(block_base64) + .map(|index| BlockLocation { + file_index: index as u32, + ziplocation: self.ziplocation.clone(), + }) + } + + pub fn contains_file_name(&self, block_base64: &str) -> bool { + self.archive.contains_file_name(block_base64) + } +} + pub struct MyCloneFileConfig { pub path: PathBuf, /// Changes after the files are indexed. From d38c232816b429403eb1cc7222a5cdb1c7ce8860 Mon Sep 17 00:00:00 2001 From: 7ERr0r <7ERr0r@users.noreply.github.com> Date: Tue, 17 Jan 2023 01:46:17 +0100 Subject: [PATCH 38/38] more cleanup --- README.md | 2 +- src/dhatprof.rs | 15 +++++++++++++ src/flags.rs | 33 ++++++++++++++++++++++++++++ src/main.rs | 55 +++++++---------------------------------------- src/ziparchive.rs | 6 ++++++ 5 files changed, 63 insertions(+), 48 deletions(-) create mode 100644 src/dhatprof.rs create mode 100644 src/flags.rs diff --git a/README.md b/README.md index e039df4..015147c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Processes files across many threads, to maximze restore speed. Usage: cargo run -- --backup-dir --restore-dir ``` -[More flags here](https://github.com/7ERr0r/duplicati-restore-rs/blob/master/src/main.rs#L31) +[More flags here](https://github.com/7ERr0r/duplicati-restore-rs/blob/master/src/flags.rs#L5) Or download the latest [binary from releases](https://github.com/7ERr0r/duplicati-restore-rs/releases) diff --git a/src/dhatprof.rs b/src/dhatprof.rs new file mode 100644 index 0000000..281c9d3 --- /dev/null +++ b/src/dhatprof.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "dhat-heap")] +pub fn start_dhat_profiler() { + std::thread::spawn(|| { + let _profiler = dhat::Profiler::new_heap(); + + std::thread::sleep(std::time::Duration::from_secs(10 * 60)); + // save profile after 10 minutes + }); + + std::thread::sleep(std::time::Duration::from_millis(200)); +} + +/// Does nothing +#[cfg(not(feature = "dhat-heap"))] +pub fn start_dhat_profiler() {} diff --git a/src/flags.rs b/src/flags.rs new file mode 100644 index 0000000..a53a672 --- /dev/null +++ b/src/flags.rs @@ -0,0 +1,33 @@ +use clap::Parser; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +pub struct RestoreFlags { + /// the location of the backup + #[arg(short, long)] + pub backup_dir: String, + + /// a location to restore to + #[arg(short, long, value_name = "FILE")] + pub restore_dir: Option, + + /// 1 thread will save and read files sequentially + #[arg(short, long, default_value_t = 4)] + pub threads_rayon: usize, + + /// displays progress bar in CLI + #[arg(short, long)] + pub progress_bar: bool, + + /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. + #[arg(long)] + pub hash_to_path: bool, + + /// true to restore windows backup on linux + #[arg(long)] + pub replace_backslash_to_slash: Option, + + /// true to verify without writing files to disk + #[arg(long)] + pub verify_only: bool, +} diff --git a/src/main.rs b/src/main.rs index d870ab6..44315ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,18 +4,22 @@ mod blockhash; mod database; mod dfileentry; mod dfiletype; +mod flags; mod hexdisplay; mod restoring; mod sorting; mod stripbom; mod ziparchive; +use crate::flags::RestoreFlags; use crate::restoring::{restore_entry, RestoreContext, RestoreParams, RestoreSummary}; use crate::sorting::sort_files_sequentially; use crate::stripbom::StripBom; + use clap::Parser; use database::*; use dfileentry::*; +use dhatprof::start_dhat_profiler; use eyre::eyre; use eyre::{Context, Result}; use pbr::ProgressBar; @@ -25,51 +29,10 @@ use std::fs::File; use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct CliArgs { - /// the location of the backup - #[arg(short, long)] - backup_dir: String, - - /// a location to restore to - #[arg(short, long, value_name = "FILE")] - restore_dir: Option, - - /// 1 thread will save and read files sequentially - #[arg(short, long, default_value_t = 4)] - threads_rayon: usize, - - /// displays progress bar in CLI - #[arg(short, long)] - progress_bar: bool, - - /// true if use additional hashmap to speed up hashed name lookup. Increases memory usage. - #[arg(long)] - hash_to_path: bool, - - /// true to restore windows backup on linux - #[arg(long)] - replace_backslash_to_slash: Option, - - /// true to verify without writing files to disk - #[arg(long)] - verify_only: bool, -} +mod dhatprof; fn main() { - #[cfg(feature = "dhat-heap")] - { - std::thread::spawn(|| { - let _profiler = dhat::Profiler::new_heap(); - - std::thread::sleep(std::time::Duration::from_secs(10 * 60)); - // save profile after 10 minutes - }); - - std::thread::sleep(std::time::Duration::from_millis(200)); - } + start_dhat_profiler(); let result = run(); match result { @@ -108,8 +71,6 @@ fn parse_dlist_file>(dlist_path: P) -> Result { let mut dlist_zip = zip::ZipArchive::new(dlist_reader)?; let filelist_name = "filelist.json"; let dlist_file = dlist_zip.by_name(filelist_name)?; - // let mut dlist_contents = Vec::new(); - // dlist_file.read_to_end(&mut dlist_contents)?; let bufrdr = BufReader::with_capacity(32 * 1024, dlist_file); let list = parse_dlist_read(bufrdr).wrap_err_with(|| { format!( @@ -137,7 +98,7 @@ fn read_manifest>(dlist_path: P) -> Result> { } fn run() -> Result<()> { - let args = CliArgs::parse(); + let args = RestoreFlags::parse(); let backup_dir = args.backup_dir.trim().to_string(); let restore_dir = if !args.verify_only { let dir = args @@ -214,7 +175,7 @@ fn run() -> Result<()> { } fn restore_all( - args: &CliArgs, + args: &RestoreFlags, params: &RestoreParams<'_>, file_entries: FileEntries, ) -> Result<()> { diff --git a/src/ziparchive.rs b/src/ziparchive.rs index bb44931..0fe326c 100644 --- a/src/ziparchive.rs +++ b/src/ziparchive.rs @@ -66,6 +66,12 @@ pub struct MyCloneFileConfig { /// Smaller buf does less redundant byte reads from disk when indexing. pub buf_capacity: AtomicU32, } + +/// Used to share ZipArchive across many threads +/// +/// Multiple ZipArchive structs would allocate too much Vec in rayon threads +/// +/// Therefore we open file again on every .clone() pub struct MyCloneFileReader { pub config: Arc, buf_reader: BufReader,