diff --git a/docs/changelog.rst b/docs/changelog.rst index a853aa2f6..4ae53a6e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) + .. _v3_39_1: 3.39.1 (2026-07-25) @@ -18,6 +25,7 @@ - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) - The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) + .. _v4_1: 4.1 (2026-07-11) diff --git a/docs/cli.rst b/docs/cli.rst index 2e506ddb7..cf241aa47 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2288,7 +2288,11 @@ If you want to see the SQL that will be executed to make the change without actu INSERT INTO "roadside_attractions_new_4033a60276b9" ("longitude", "latitude", "id", "name") SELECT "longitude", "latitude", "pk", "name" FROM "roadside_attractions"; DROP TABLE "roadside_attractions"; + PRAGMA legacy_alter_table=ON; ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; + PRAGMA legacy_alter_table=OFF; + +Tables that are referenced by views can be transformed - the view definitions are left unchanged, see :ref:`python_api_transform_views` for details. .. note:: In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` diff --git a/docs/python-api.rst b/docs/python-api.rst index 43b734dda..53a47ddd4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1986,6 +1986,17 @@ A bare column name drops any foreign key that column participates in, including Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. +.. _python_api_transform_views: + +Tables referenced by views +-------------------------- + +Tables that are referenced by views can be safely transformed - the view definitions are left byte-for-byte unchanged, and views continue to read from the live table even when ``keep_table=`` is used to keep a copy of the original around. + +A view that references a column which the transform renamed or dropped will remain defined but will raise a ``no such column`` error when it is next queried. This is inherent to SQLite views, whose SQL is stored as text - if you rename or drop columns that a view depends on you should update that view definition yourself. + +To achieve this, the SQL produced by ``transform_sql()`` turns on ``PRAGMA legacy_alter_table`` for its ``ALTER TABLE ... RENAME TO`` statements, then restores the pragma to the value it had when the SQL was generated - without this, SQLite would attempt to rewrite references to the renamed table in every view definition, which fails when a view references the table that was just dropped. + .. _python_api_transform_sql: Custom transformations with .transform_sql() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9a001231e..1b2596974 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2824,17 +2824,34 @@ def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) - # Drop (or keep) the old table + # Drop (or keep) the old table, then rename the new one into place. + # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to + # the renamed table in every view definition, which fails if a view + # references the table that was just dropped - and with keep_table= + # would silently repoint views at the backup table. These renames are + # an implementation detail of transform(), so use legacy_alter_table + # to leave view definitions untouched, restoring the connection's + # current value afterwards. + legacy_alter_table_row = self.db.execute("PRAGMA legacy_alter_table").fetchone() + legacy_alter_table_was_on = bool( + legacy_alter_table_row and legacy_alter_table_row[0] + ) if keep_table: + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" ) else: sqls.append(f"DROP TABLE {quote_identifier(self.name)};") - # Rename the new one + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" ) + sqls.append( + "PRAGMA legacy_alter_table={};".format( + "ON" if legacy_alter_table_was_on else "OFF" + ) + ) # Re-add existing indexes for index in self.indexes: if index.origin != "pk": diff --git a/tests/test_transform.py b/tests/test_transform.py index b9ee12682..7874421fd 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -16,7 +16,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -26,7 +28,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -36,7 +40,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Drop a column @@ -46,7 +52,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Convert type AND rename column @@ -56,7 +64,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key @@ -66,7 +76,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT PRIMARY KEY\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key to a compound pk @@ -76,7 +88,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT,\n PRIMARY KEY ("age", "name")\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Remove primary key, creating a rowid table @@ -86,7 +100,9 @@ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Keeping the table @@ -95,8 +111,10 @@ [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs" RENAME TO "kept_table";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -139,7 +157,9 @@ def tracer(sql, params): 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -149,7 +169,9 @@ def tracer(sql, params): 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -159,7 +181,9 @@ def tracer(sql, params): 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Make ID a primary key @@ -169,7 +193,9 @@ def tracer(sql, params): 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -903,3 +929,139 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." in str(excinfo.value) ) + + +def test_transform_preserves_view(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/831 + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(rename={"name": "title"}) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + + +@pytest.mark.parametrize( + "transform_params", + [ + {"types": {"name": int}}, + {"pk": "name"}, + {"add_foreign_keys": [("other_id", "other", "id")]}, + {"drop_foreign_keys": ["other_id"]}, + ], +) +def test_transform_variants_preserve_view(fresh_db, transform_params): + # Covers retyping, changing primary key and foreign key modifications, + # with a view whose columns are untouched by the transform + fresh_db["other"].insert({"id": 1}, pk="id") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id") + if "drop_foreign_keys" in transform_params: + dogs.transform(add_foreign_keys=[("other_id", "other", "id")]) + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(**transform_params) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_view_referencing_renamed_column(fresh_db): + # The view survives but querying it raises "no such column" - inherent + # to SQLite views, whose SQL is stored as text + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(rename={"name": "title"}) + with pytest.raises(OperationalError, match="no such column"): + fresh_db.execute("select * from dogs_view") + + +def test_transform_view_on_view(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view v1 as select id, name from dogs") + fresh_db.execute("create view v2 as select name from v1") + sqls_before = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + dogs.transform(types={"id": str}) + sqls_after = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + assert sqls_before == sqls_after + assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}] + + +def test_transform_keep_table_does_not_repoint_view(fresh_db): + # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup + # step would rewrite the view to select from "dogs_backup" + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(types={"name": str}, keep_table="dogs_backup") + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert "dogs_backup" not in view_sql + # View reads from the live table, not the frozen backup + dogs.insert({"id": 2, "name": "Pancakes"}) + assert list(fresh_db["dogs_view"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"}, + ] + + +def test_transform_sql_standalone_statements_work_with_view(fresh_db): + # The documented "run these statements yourself" workflow should be + # standalone-correct, so the pragmas must come from transform_sql() + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-3] == "PRAGMA legacy_alter_table=ON;" + assert sqls[-2] == 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";' + assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;" + for sql in sqls: + fresh_db.execute(sql) + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_with_view_in_open_transaction(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + with fresh_db.conn: + fresh_db.execute("insert into dogs (id, name) values (2, 'Pancakes')") + dogs.transform(rename={"name": "title"}) + assert dogs.columns_dict == {"id": int, "title": str} + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql == "CREATE VIEW dogs_view as select id, name from dogs" + + +def test_transform_restores_legacy_alter_table_setting(fresh_db): + if sqlite3.sqlite_version_info < (3, 25, 0): + pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + # Default is OFF, reset to OFF afterwards + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 0 + # If the connection has it ON, it should be restored to ON + fresh_db.execute("PRAGMA legacy_alter_table=ON") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-1] == "PRAGMA legacy_alter_table=ON;" + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1