sqlite-diffable load command, closes #3

Author: simonw

README.md

@@ -10,7 +10,7 @@ Tools for dumping/loading a SQLite database to diffable directory structure
 
     pip install sqlite-diffable
 
-## Usage
+## Dumping a database
 
 Given a SQLite database called `fixtures.db` containing a table `facetable`, the following will dump out that table to the `out/` directory:
 
@@ -20,6 +20,12 @@ To dump out every table in that database, use `--all`:
 
     sqlite-diffable dump fixtures.db out/ --all
 
+## Loading a database
+
+To load a previously dumped database, run the following:
+
+    sqlite-diffable load restored.db out/
+
 ## Demo
 
 The repository at [simonw/simonwillisonblog-backup](https://github.com/simonw/simonwillisonblog-backup) contains a backup of the database on my blog, https://simonwillison.net/ - created using this tool.

sqlite_diffable/cli.py

@@ -13,19 +13,32 @@ def cli():
 
 @cli.command()
 @click.argument(
-    "path", type=click.Path(exists=True, file_okay=True, dir_okay=False), required=True
+    "dbpath",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+    required=True,
 )
 @click.argument(
     "output", type=click.Path(file_okay=False, dir_okay=True), required=True
 )
 @click.argument("tables", nargs=-1, required=False)
-@click.option("--all", is_flag=True)
-def dump(path, output, tables, all):
+@click.option("--all", is_flag=True, help="Dump all tables")
+def dump(dbpath, output, tables, all):
+    """
+    Dump a SQLite database out as flat files in the directory
+
+    Usage:
+
+        sqlite-diffable dump my.db output/ --all
+
+    --all dumps ever table. Or specify tables like this:
+
+        sqlite-diffable dump my.db output/ entries tags
+    """
     if not tables and not all:
         raise click.ClickException("You must pass --all or specify some tables")
     output = pathlib.Path(output)
     output.mkdir(exist_ok=True)
-    conn = sqlite_utils.Database(path)
+    conn = sqlite_utils.Database(dbpath)
     if all:
         tables = conn.table_names()
     for table in tables:
@@ -48,3 +61,38 @@ def dump(path, output, tables, all):
                 indent=4,
             )
         )
+
+
+@cli.command()
+@click.argument(
+    "dbpath",
+    type=click.Path(file_okay=True, allow_dash=False, dir_okay=False),
+)
+@click.argument(
+    "directory",
+    type=click.Path(file_okay=False, dir_okay=True),
+)
+def load(dbpath, directory):
+    """
+    Load flat files from a directory into a SQLite database
+
+    Usage:
+
+        sqlite-diffable load my.db output/
+    """
+    db = sqlite_utils.Database(dbpath)
+    directory = pathlib.Path(directory)
+    metadatas = directory.glob("*.metadata.json")
+    for metadata in metadatas:
+        info = json.loads(metadata.read_text())
+        columns = info["columns"]
+        schema = info["schema"]
+        db.execute(schema)
+        # Now insert the rows
+        ndjson = metadata.parent / metadata.stem.replace(".metadata", ".ndjson")
+        rows = (
+            dict(zip(columns, json.loads(line)))
+            for line in ndjson.open()
+            if line.strip()
+        )
+        db[info["name"]].insert_all(rows)

tests/test_dump.py

@@ -1,5 +1,6 @@
 from click.testing import CliRunner
 from sqlite_diffable import cli
+import sqlite_utils
 import json
 
 
@@ -8,7 +9,7 @@ def test_dump(one_table_db, tmpdir):
     result = CliRunner().invoke(
         cli.cli, ["dump", one_table_db, str(output_dir), "one_table"]
     )
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     # out/ should now have a single file in it
     ndjson = output_dir / "one_table.ndjson"
     metadata = output_dir / "one_table.metadata.json"
@@ -29,8 +30,30 @@ def test_dump_all(two_tables_db, tmpdir):
     result = CliRunner().invoke(
         cli.cli, ["dump", two_tables_db, str(output_dir), "--all"]
     )
-    assert 0 == result.exit_code, result.output
+    assert result.exit_code == 0, result.output
     assert (output_dir / "one_table.ndjson").exists()
     assert (output_dir / "one_table.metadata.json").exists()
     assert (output_dir / "second_table.ndjson").exists()
     assert (output_dir / "second_table.metadata.json").exists()
+
+
+def test_load(two_tables_db, tmpdir):
+    output_dir = tmpdir / "out"
+    restore_db = tmpdir / "restore.db"
+    result = CliRunner().invoke(
+        cli.cli, ["dump", str(two_tables_db), str(output_dir), "--all"]
+    )
+    assert result.exit_code == 0, result.output
+    # Now load it again
+    result2 = CliRunner().invoke(cli.cli, ["load", str(restore_db), str(output_dir)])
+    assert result2.exit_code == 0, result2.output
+    db = sqlite_utils.Database(str(restore_db))
+    assert set(db.table_names()) == {"second_table", "one_table"}
+    assert list(db["one_table"].rows) == [
+        {"id": 1, "name": "Stacey"},
+        {"id": 2, "name": "Tilda"},
+        {"id": 3, "name": "Bartek"},
+    ]
+    assert list(db["second_table"].rows) == [
+        {"id": 1, "name": "Cleo"},
+    ]