diff --git a/README.md b/README.md index 9a1835a..c68cecb 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ db = TypeDAL("sqlite:memory") # db = TypeDAL("mysql://user:password@localhost:3306/mydb") # ... + @db.define() class User(TypedTable): name: str @@ -143,11 +144,12 @@ db = TypeDAL(...) ```python -db.define_table("table_name", - Field("fieldname", "string", required=True), - Field("otherfield", "float"), - Field("yet_another", "text", default="Something") - ) +db.define_table( + "table_name", + Field("fieldname", "string", required=True), + Field("otherfield", "float"), + Field("yet_another", "text", default="Something"), +) ``` @@ -236,7 +238,6 @@ all_rows = TableName.collect() # or .all() rows = TableName.select(Tablename.id).where(TableName.id > 5).where(TableName.id < 50).collect() # one: row = TableName(id=1) # or .where(...).first() - ``` @@ -301,11 +302,11 @@ These helpers are useful for scenarios where direct access to the PyDAL objects An example of this is when you need to do a `db.commit()` but you can't import `db` directly: ```python -from typedal.helpers import get_db #, get_table, get_field +from typedal.helpers import get_db # , get_table, get_field MyTable.insert(...) db = get_db(MyTable) -db.commit() # this is usually done automatically but sometimes you want to manually commit. +db.commit() # this is usually done automatically but sometimes you want to manually commit. ``` ## Caveats diff --git a/docs/1_getting_started.md b/docs/1_getting_started.md index 8be5550..1a2a5e7 100644 --- a/docs/1_getting_started.md +++ b/docs/1_getting_started.md @@ -20,6 +20,7 @@ pip install typedal[py4web] ```python from typedal import TypeDAL + # or, if in py4web: from typedal.for_py4web import TypeDAL @@ -55,15 +56,11 @@ Or use the `placeholders` argument with positional or named parameters: ```python # Positional -rows = db.executesql( - "SELECT * FROM some_table WHERE name = %s AND age > %s", - placeholders=[name, 18] -) +rows = db.executesql("SELECT * FROM some_table WHERE name = %s AND age > %s", placeholders=[name, 18]) # Named rows = db.executesql( - "SELECT * FROM some_table WHERE name = %(name)s AND age > %(age)s", - placeholders={"name": name, "age": 18} + "SELECT * FROM some_table WHERE name = %(name)s AND age > %(age)s", placeholders={"name": name, "age": 18} ) ``` @@ -73,14 +70,11 @@ By default, `executesql()` returns rows as tuples. To map results to specific fi Field/TypedField objects) or `colnames` (takes column name strings): ```python -rows = db.executesql( - "SELECT id, name FROM some_table", - colnames=["id", "name"] -) +rows = db.executesql("SELECT id, name FROM some_table", colnames=["id", "name"]) rows = db.executesql( "SELECT id, name FROM some_table", - fields=[some_table.id, some_table.name] # Requires table definition + fields=[some_table.id, some_table.name], # Requires table definition ) ``` diff --git a/docs/2_defining_tables.md b/docs/2_defining_tables.md index 36a0cc0..efa22f8 100644 --- a/docs/2_defining_tables.md +++ b/docs/2_defining_tables.md @@ -6,7 +6,7 @@ The syntax for creating a table is very different, but built on the same princip from pydal import Field # pydal: -db.define_table('my_table', Field('my_field')) +db.define_table("my_table", Field("my_field")) ``` ```python @@ -121,8 +121,7 @@ from typedal import TypedTable from typedal.types import OpRow, Reference, Set -class MyTable(TypedTable): - ... +class MyTable(TypedTable): ... def my_before_insert(row: MyTable): diff --git a/docs/3_building_queries.md b/docs/3_building_queries.md index 76a8e23..dd1fc39 100644 --- a/docs/3_building_queries.md +++ b/docs/3_building_queries.md @@ -84,7 +84,7 @@ Here you can enter any number of fields as arguments: database columns by name ( other (e.g. Table.ALL), or Expression objects. ```python -Person.select('id', Person.name, Person.ALL) # defaults to Person.ALL if select is omitted. +Person.select("id", Person.name, Person.ALL) # defaults to Person.ALL if select is omitted. ``` You can also specify extra options as keyword arguments. Supported options are: `orderby`, `groupby`, `limitby`, @@ -131,10 +131,7 @@ Person.where(expr).select() # Named arguments expr = db.sql_expression( - "EXTRACT(year FROM %(date_col)s) = %(year)s", - date_col="created_at", - year=2023, - output_type="boolean" + "EXTRACT(year FROM %(date_col)s) = %(year)s", date_col="created_at", year=2023, output_type="boolean" ) Person.where(expr).select() ``` @@ -152,7 +149,7 @@ By default, the `method` defined in the relationship is used. This can be overwritten with the `method` keyword argument (left or inner) ```python -Person.join('articles', method='inner') # will only yield persons that have related articles +Person.join("articles", method="inner") # will only yield persons that have related articles ``` For more details about relationships and joins, see [4. Relationships](./4_relationships.md). @@ -277,15 +274,18 @@ class User(TypedTable): password_hash: str is_active: bool + class PublicUser(TypedTable): id: int email: str is_active: bool profile_url: str | None = None + def enrich_profile_url(row: PublicUser, _raw): row.profile_url = f"/users/{row.id}" + rows = User.where(is_active=True).collect_into( PublicUser, # note: `init` is optional: diff --git a/docs/4_relationships.md b/docs/4_relationships.md index f3a4989..db6061f 100644 --- a/docs/4_relationships.md +++ b/docs/4_relationships.md @@ -20,8 +20,10 @@ class Post(TypedTable): author: Author -authors_with_roles = Author.join('roles').collect() -posts_with_author = Post.join().collect() # join can be called without arguments to join all relationships (in this case only 'author') +authors_with_roles = Author.join("roles").collect() +posts_with_author = ( + Post.join().collect() +) # join can be called without arguments to join all relationships (in this case only 'author') post_deep = Post.join("author.roles").collect() # nested relationship, accessible via post.author.roles ``` @@ -127,32 +129,39 @@ owner: "User" Setting up a relationship that uses a junction/pivot table is slightly harder. ```python - # with `unique_alias()` which is better if you have multiple joins: + @db.define() class Post(TypedTable): title: str author: Author - tags = relationship(list["Tag"], on=lambda post, tag: [ - # post and tag already have a unique alias, create one for tagged here: - tagged := Tagged.unique_alias(), - tagged.on(tagged.post == post.id), - tag.on(tag.id == tagged.tag), - ]) + tags = relationship( + list["Tag"], + on=lambda post, tag: [ + # post and tag already have a unique alias, create one for tagged here: + tagged := Tagged.unique_alias(), + tagged.on(tagged.post == post.id), + tag.on(tag.id == tagged.tag), + ], + ) # without unique alias: + @db.define() class Tag(TypedTable): name: str - posts = relationship(list["Post"], on=lambda tag, posts: [ - Tagged.on(Tagged.tag == tag.id), - posts.on(posts.id == Tagged.post), - ]) + posts = relationship( + list["Post"], + on=lambda tag, posts: [ + Tagged.on(Tagged.tag == tag.id), + posts.on(posts.id == Tagged.post), + ], + ) @db.define() diff --git a/docs/5_py4web.md b/docs/5_py4web.md index e0a3fa6..64869aa 100644 --- a/docs/5_py4web.md +++ b/docs/5_py4web.md @@ -8,10 +8,7 @@ This library also has some py4web/web2py-specific enhancements. # common.py from typedal.for_py4web import DAL -db = DAL( - settings.DB_URI, - ... -) +db = DAL(settings.DB_URI, ...) ``` This version of the `DAL` is also a py4web Fixture that manages database connections `on_request`, just as py4web's own @@ -27,6 +24,7 @@ from .common import db # you can now customize auth user: + class AuthUser(_AuthUser): bookmarks = relationship(list["Bookmark"], ...) @@ -35,7 +33,6 @@ db.define(AuthUser, redefine=True) # or if you don't want to customize auth user: setup_py4web_tables(db) - ``` TypeDAL also provides an `AuthUser` class based on `db.auth_user`. @@ -53,6 +50,7 @@ from .common import db # you can now customize auth user: + class AuthUser(_AuthUser): bookmarks = relationship(list["Bookmark"], ...) diff --git a/docs/8_mixins.md b/docs/8_mixins.md index 91c0af5..43b0f5b 100644 --- a/docs/8_mixins.md +++ b/docs/8_mixins.md @@ -23,6 +23,7 @@ class MyTable(TypedTable, TimestampsMixin): # Define your table fields here pass + # Now, whenever you create or update a record in MyTable, the 'created_at' and 'updated_at' timestamps will be automatically managed. ``` @@ -44,6 +45,7 @@ class MyTable(TypedTable, SlugMixin, slug_field="title"): title: str # Assuming 'title' is a field in your table # Define other fields here + # Now, whenever you insert a record into MyTable, the 'slug' field will be automatically generated based on the 'title' field. ``` @@ -101,6 +103,7 @@ from fastapi import FastAPI app = FastAPI() + @app.get("/books/{book_id}") def get_book(book_id: int) -> Book: return Book.where(id=book_id).join("author").first() @@ -154,9 +157,10 @@ class HasImageMixin(Mixin): # Now you can use HasImageMixin in your table definitions along with other mixins or base classes. + class Article(TypedTable, TimestampsMixin, HasImageMixin): title: str - + # this could also be a class method of Timestamps Mixin: @classmethod def recently_updated(cls, hours: int = 24) -> QueryBuilder[t.Self]: @@ -164,16 +168,13 @@ class Article(TypedTable, TimestampsMixin, HasImageMixin): cutoff = dt.datetime.now() - dt.timedelta(hours=hours) return QueryBuilder(cls).where(cls.updated_at >= cutoff) + # Retrieve a record and use the custom method article = Article(id=1) article.img() # -> # Use the classmethod to get recently updated articles -recent_articles = ( - Article.recently_updated(hours=12) - .where(published=True) - .collect() -) +recent_articles = Article.recently_updated(hours=12).where(published=True).collect() ``` > **Note:** The `img()` example uses py4web utilities (URL, IMG), but the mixin itself works identically in any setup. diff --git a/docs/9_memoization.md b/docs/9_memoization.md index 2f50020..fd51c2b 100644 --- a/docs/9_memoization.md +++ b/docs/9_memoization.md @@ -18,9 +18,10 @@ def process_articles(articles: TypedRows[Article]) -> dict: # dummy example, normally you'd use .join() of course for article in articles: comments = Comment.where(article=article).collect() - result[article.id] = comments + result[article.id] = comments return result + articles = Article.where(published=True).collect() result, status = db.memoize(process_articles, articles) @@ -53,6 +54,7 @@ When any tracked row is updated, inserted, or deleted, the cached result is inva def something_slow(): return list(User.join()) + result, status = db.memoize(something_slow) assert status == "fresh" @@ -114,9 +116,11 @@ TypeDAL provides `before_collect`/`before_execute` and `after_collect`/`after_ex def print_query(qb: QueryBuilder): print("going to run", qb.to_sql()) + def print_duration(_qb: QueryBuilder, rows, _raw): print("took", rows.metadata["select_duration"]) + db.before_collect.append(print_query) db.after_collect.append(print_duration) @@ -143,8 +147,7 @@ If you need to disable cache invalidation hooks for a specific table: ```python @db.define(cache_dependency=False) -class SpecialTable(TypedTable): - ... +class SpecialTable(TypedTable): ... ``` **Warning:** Disabling this may break caching functionality for queries involving this table. diff --git a/pyproject.toml b/pyproject.toml index 31b753f..6e2b1a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,14 @@ typescript = [ "pydal2sql>=1.3.5", ] +postgres-async = [ + "psycopg[binary,pool]", +] + +sqlite-async = [ + "aiosqlite", +] + all = [ "py4web", "typtyp < 1", @@ -82,6 +90,9 @@ all = [ "questionary", "tomlkit", "pydal2sql[all]>=1.3.5", + # async: + "psycopg[binary,pool]", + "aiosqlite", ] dev = [ @@ -93,11 +104,14 @@ dev = [ "python-semantic-release < 8", # "pytest-mypy-testing", "pytest-typing", + "pytest-asyncio", "pyright < 1.1.400", "contextlib-chdir", "testcontainers", "pydantic < 3", + "psycopg[binary,pool]", "psycopg2-binary", + "aiosqlite", # depends on -> "requests<2.32", # mypy: @@ -134,6 +148,10 @@ upload_to_repository = false upload_to_release = false build_command = "hatch build" +[tool.edwh.test] +# measure coverage over src/ only, matching su6 +directory = "src" + ### required in every su6 pyproject: ### [tool.su6] directory = "src" @@ -194,6 +212,14 @@ target-version = "py314" line-length = 120 extend-exclude = ["*.bak/", "venv*/", "tests/", "example_*.py"] + +[tool.ty.src] +include = ["src"] + +[tool.ty.environment] +python = "./venv" +python-version = "3.14" + [tool.ruff.lint] select = [ diff --git a/src/typedal/__init__.py b/src/typedal/__init__.py index 446dea2..fccd7fd 100644 --- a/src/typedal/__init__.py +++ b/src/typedal/__init__.py @@ -2,6 +2,11 @@ TypeDAL Library. """ +from .async_execution import ( + ConcurrentTransactionError, + TransactionBoundaryError, + TransactionSplitError, +) from .core import TypeDAL from .fields import TypedField from .helpers import sql_expression @@ -15,13 +20,16 @@ try: from .for_py4web import DAL as P4W_DAL except ImportError: # pragma: no cover - P4W_DAL = None # type: ignore + P4W_DAL = None __all__ = [ + "ConcurrentTransactionError", "PaginatedRows", "QueryBuilder", "Ref", "Relationship", + "TransactionBoundaryError", + "TransactionSplitError", "TypeDAL", "TypedField", "TypedRows", diff --git a/src/typedal/async_execution.py b/src/typedal/async_execution.py new file mode 100644 index 0000000..2f813b8 --- /dev/null +++ b/src/typedal/async_execution.py @@ -0,0 +1,1084 @@ +""" +Backend-specific plumbing for TypeDAL's async execution path. + +`TypeDAL` (core.py) owns the actual `_async` methods (`select_async`, `_get_async_pool`, ...) - +those are legitimately DAL-instance behavior. This module only holds the per-backend detail of +"how do you get an async connection for this dbengine", kept out of core.py so that stays about +the `TypeDAL` class itself, not about psycopg/aiosqlite specifics. + +One factory per backend, registered by pydal's `adapter.dbengine` name in +`_ASYNC_POOL_FACTORIES`, rather than an if/elif chain - adding a new backend (e.g. MySQL) means +adding a function + a registry entry here, not editing branching logic in `TypeDAL._get_async_pool`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import typing as t + +import pydal.objects +from pydal.helpers.classes import ExecutionHandler + +if t.TYPE_CHECKING: + from pydal.adapters.base import SQLAdapter + + from .core import TypeDAL + + +# What pydal's `adapter._insert()` leaves behind to record whether the statement it just built +# carries a RETURNING clause: `(table._id, 1)` when it does, `None` when it does not +# (adapters/postgres.py). Backends without the concept never set it at all, hence None. +type LastInsert = tuple[pydal.objects.Field, int] | None + +# SQL verbs that open a transaction on whichever connection runs them, used by +# `SyncTransactionTracker` to decide whether a statement pydal just ran left uncommitted work. +# DDL is left out on purpose: `db.define()` migrates on the sync connection, and treating that +# as pending work would make the first `_async` call after any table definition raise. +# +# A text prefix is a weak test - a leading comment or a CTE hides the verb - and the async side +# deliberately no longer uses it: it asks the connection instead, see +# `UNCOMMITTED_WORK_STRATEGIES`. The sync side cannot do the same without also catching +# migration DDL, so it stays on the prefix for now. +WRITE_STATEMENTS = ("INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE", "TRUNCATE") + +# How long a file-backed SQLite connection waits for another one's write to finish before +# giving up with `database is locked`. SQLite allows a single writer at a time, so per-task +# connections queue here rather than failing outright; 5s is aiosqlite's own default order of +# magnitude and well past any statement a request should be issuing. +SQLITE_BUSY_TIMEOUT_MS = 5000 + +# Ceiling on concurrent per-task Postgres connections from one `TypeDAL`. Has to exceed 1 or +# `PostgresAsyncPool`'s per-task checkout deadlocks as soon as two tasks overlap; kept modest +# because every `TypeDAL` in the process draws from the same server-side max_connections. +POSTGRES_POOL_MAX_SIZE = 10 + + +class TransactionBoundaryError(RuntimeError): + """ + Base for the two ways a caller can end up on the wrong side of a transaction boundary. + + Both subclasses exist for the same reason: the alternative to raising is a silently wrong + answer, and this class of bug only shows under concurrency, which is the worst place to + find it. Catch this to handle either. + """ + + +class TransactionSplitError(TransactionBoundaryError): + """ + Raised when sync and async work would be split across the two connections a `TypeDAL` has. + + pydal drives Postgres with psycopg2 and SQLite with sqlite3, both synchronous; the `_async` + path needs psycopg3-async and aiosqlite. Those are separate connections and therefore + separate transactions, so uncommitted work on one is invisible to the other. Within a + single request that is read-your-own-writes quietly disappearing: on Postgres the second + path simply does not see the row, and on SQLite it blocks on the table lock instead. + + Rather than let either happen, both paths refuse to run while the other holds an open + transaction. Commit or roll back the side you finished with before using the other one. + + Warning-and-continuing was considered and does not survive contact with SQLite. Measured on + the same scenario: Postgres returns the committed rows (wrong but warnable), a plain SQLite + read raises `database table is locked` and cannot proceed at all, and SQLite with + `PRAGMA read_uncommitted=1` returns *more* rows than Postgres - including ones a rollback + then deletes. Three answers to identical code, two of them silent. Raising is the only + behaviour both backends can actually share. + """ + + +class ConcurrentTransactionError(TransactionBoundaryError): + """ + Raised when two asyncio tasks would share one transaction on a `sqlite:memory` database. + + Postgres and file-backed SQLite both hand each task its own connection, so their + transactions are independent. `sqlite:memory` cannot: a second connection only reaches the + same database through shared-cache mode, which answers a concurrent writer with + SQLITE_LOCKED. One connection means one transaction, and sharing it means one task's + `rollback_async()` destroys another task's uncommitted rows. + + So the second task is refused instead. pydal's own synchronous connections hit this same + wall between two threads on one `sqlite:memory` - this raises deliberately, and says why, + where pydal surfaces the driver's `database table is locked`. + """ + + +class SyncTransactionTracker(ExecutionHandler): + """ + Records whether pydal's own connection has uncommitted writes, and refuses to run a sync + statement while the async connection has some (see `TransactionSplitError`). + + An `ExecutionHandler` rather than wrappers around `insert`/`update`/`delete`, because this + has to see *every* statement reaching the adapter - `executesql()`, pydal internals and + anything a caller reaches around TypeDAL for included - and `DAL.execution_handlers` is + pydal's own supported seam for that (it is where `TimingHandler` lives). + + Flags live on the `TypeDAL`, not here: pydal builds a handler instance per execution, so + this object is the wrong place to keep anything that has to outlive one statement. + """ + + def before_execute(self, command: str) -> None: + """ + Check the async side is settled, then note whether this statement opens a transaction. + """ + db = getattr(self.adapter, "db", None) + if db is None: # pragma: no cover - adapter detached during close() + return + + # TypeDAL owns the settle-then-check decision, so the sync side cannot accidentally ask + # the predicate without first reclaiming an abandoned sqlite:memory transaction. + if db._has_pending_async_writes(): + raise TransactionSplitError( + "The async connection has uncommitted writes, which this synchronous statement " + "would not see. Call `await db.commit_async()` or `await db.rollback_async()` " + "first. If those writes belong to another task, wait for it to settle them: " + "neither call ends a transaction this task does not own.", + ) + + if command.lstrip().upper().startswith(WRITE_STATEMENTS): + db._sync_pending = True + + +class AsyncCursor(t.Protocol): + """ + The slice of a psycopg / aiosqlite cursor that the async execution path actually uses. + + A Protocol rather than the real driver cursor types, because both drivers are *optional* + dependencies (`typedal[postgres-async]` / `typedal[sqlite-async]`): naming either one in a + signature would make type-checking TypeDAL require it to be installed. Structural typing + gets the checking without the dependency. + + Read-only properties rather than plain attributes so that both drivers match - psycopg and + aiosqlite both expose `rowcount`/`lastrowid`/`description` as properties, and a Protocol + declaring them as mutable attributes would reject exactly that. + """ + + @property + def rowcount(self) -> int: ... + + @property + def lastrowid(self) -> int | None: ... + + @property + def description(self) -> t.Any: ... + + async def execute(self, sql: str, parameters: t.Any = ..., /) -> t.Any: ... + + async def fetchone(self) -> t.Any: ... + + # `Iterable`, not `Sequence`: aiosqlite declares `fetchall() -> Iterable[sqlite3.Row]` + # (aiosqlite/cursor.py), so requiring a Sequence here would reject it. + async def fetchall(self) -> t.Iterable[t.Any]: ... + + +class AsyncConnection(t.Protocol): + """ + The slice of a psycopg / aiosqlite connection the async execution path uses. Same reasoning + as `AsyncCursor`. + + `cursor()` is typed as returning a context manager, not a cursor or an awaitable, because + that is the one shape both drivers share: psycopg's `cursor()` returns an `AsyncCursor` + that doubles as an async context manager, while aiosqlite's is decorated to return a + `Result[Cursor]` (aiosqlite/context.py) which is both awaitable *and* an async context + manager. `async with conn.cursor() as cur` is what works for both. + """ + + def cursor(self) -> t.AsyncContextManager[AsyncCursor]: ... + + async def commit(self) -> None: ... + + async def rollback(self) -> None: ... + + async def close(self) -> None: ... + + +class AsyncConnectionPool(t.Protocol): + """ + Common shape `select_async()` etc. need from either a real connection pool (Postgres) or a + single-connection stand-in (SQLite). + + `commit()`/`rollback()` are part of this shape (not left to `TypeDAL.commit_async()` to + figure out per backend) because what they act on genuinely differs: `PostgresAsyncPool` + ends the transaction on the connection checked out for *this task* and returns it to the + pool, while `SqliteAsyncConnection` ends the one transaction there is. Keeping both behind + the same two methods keeps that difference out of core.py. + + Either way `connection()` leaves the transaction open, so `_async` writes obey pydal's + contract: nothing is durable until the caller commits. + """ + + def connection(self) -> t.AsyncContextManager[AsyncConnection]: ... + + async def commit(self) -> None: ... + + async def rollback(self) -> None: ... + + async def close(self) -> None: ... + + def settle_abandoned_sync(self) -> bool: ... + + +def _spawn_reclaim(coro: t.Coroutine[t.Any, t.Any, None], tasks: "set[asyncio.Task[None]]") -> None: + """ + Run a reclaim coroutine on the running loop, holding a reference until it finishes. + + `add_done_callback` is synchronous, so the actual rollback/close has to be scheduled. The + reference is the point: the event loop only keeps a *weak* one to a task, so a bare + `create_task(...)` can be garbage-collected part-way through its rollback. That surfaces as + a connection which is never handed back - only under load, and never twice in the same + place. + + Best-effort, like everything on the reclaim path: with no running loop there is nothing to + schedule on, and closing the pool is what reclaims the connection instead. + """ + try: + task = asyncio.get_running_loop().create_task(coro) + except RuntimeError: + # no running loop; closing the coroutine keeps it from warning about never being awaited + coro.close() + return + + tasks.add(task) + task.add_done_callback(tasks.discard) + + +class PostgresAsyncPool: + """ + Wrap `psycopg_pool.AsyncConnectionPool` with one connection per asyncio task. + + A task keeps its connection until `commit()` or `rollback()` so its `_async` calls share a + transaction. Abandoned task connections are rolled back and returned by a done-callback. + """ + + def __init__(self, pool: t.Any) -> None: + self._pool = pool + self._current: contextvars.ContextVar[t.Any] = contextvars.ContextVar( + f"typedal_async_conn_{id(self):x}", + default=None, + ) + self._checked_out: set[t.Any] = set() + # see `_spawn_reclaim()` - without this the reclaim tasks can be collected mid-flight. + self._reclaim_tasks: "set[asyncio.Task[None]]" = set() + + def _own_connection(self) -> t.Any: + """ + The connection this task acquired, or None - including when the value it can see was + acquired by a different task. + + That last part is the whole reason the entry stores its owner. A `ContextVar` set in a + parent is *copied into* every task the parent later spawns, so two coroutines under one + `asyncio.gather()` would both see the parent's connection and hand it around as if it + were theirs - one task's commit closing the transaction the other was still writing to. + Isolation only holds if an inherited entry is treated as absent. + """ + entry = self._current.get() + if entry is None: + return None + + owner, conn = entry + return conn if owner is asyncio.current_task() else None + + async def _acquire(self) -> t.Any: + """ + This task's connection, checking one out of the pool on first use. + """ + if (conn := self._own_connection()) is not None: + return conn + + conn = await self._pool.getconn() + self._current.set((asyncio.current_task(), conn)) + self._checked_out.add(conn) + + if task := asyncio.current_task(): + task.add_done_callback(lambda _task: self._reclaim(conn)) + + return conn + + def _reclaim(self, conn: t.Any) -> None: + """ + Return a connection its task never ended the transaction on (see the class docstring). + + Sync, because that is all `add_done_callback` can be, so the actual work is scheduled. + Everything here is best-effort: the loop may already be shutting down, in which case + closing the pool is what reclaims the connection instead. + """ + if conn not in self._checked_out: + # the ordinary case - commit() or rollback() already handed it back + return + + async def _rollback_and_return() -> None: + # Claim before doing anything, and claim by *removing* from `_checked_out`. The + # check-and-discard runs before the first await, so it is atomic against the other + # two paths that also return connections (`_release` and `close`), and whoever + # claims first is the only one that acts. Holding membership across the await + # instead let `close()` return the same connection concurrently, which psycopg + # answers with `can't return connection to pool, it doesn't come from any pool`. + if conn not in self._checked_out: + return + + self._checked_out.discard(conn) + + with contextlib.suppress(Exception): + await conn.rollback() + + try: + await self._pool.putconn(conn) + except Exception: + # The pool is gone or refused it, so this connection can never be handed back. + # Close it rather than re-tracking it: `close()` has already run by the time + # that happens, so nothing would ever drain the set again and the socket would + # stay open for the life of the process - which exhausts the server's + # max_connections one abandoned task at a time. + with contextlib.suppress(Exception): + await conn.close() + + _spawn_reclaim(_rollback_and_return(), self._reclaim_tasks) + + async def _release(self, conn: t.Any) -> None: + """ + Hand this task's connection back, after its transaction has been ended. + + Handed the connection rather than reading the `ContextVar` again: both callers have + just read it to decide there was a transaction to end at all, and a second read is one + more opportunity for the two to disagree about which connection this is. + """ + self._current.set(None) + self._checked_out.discard(conn) + await self._pool.putconn(conn) + + @contextlib.asynccontextmanager + async def connection(self) -> t.AsyncIterator[AsyncConnection]: + # no commit and no rollback on exit: whether this statement stands is the caller's + # call, made via commit()/rollback(), exactly as it is on pydal's sync connection. + yield t.cast(AsyncConnection, await self._acquire()) + + def settle_abandoned_sync(self) -> bool: + """Per-task backends reclaim abandoned connections through their own done-callback.""" + return True + + async def commit(self) -> None: + if (conn := self._own_connection()) is None: + return + + await conn.commit() + await self._release(conn) + + async def rollback(self) -> None: + if (conn := self._own_connection()) is None: + return + + await conn.rollback() + await self._release(conn) + + async def close(self) -> None: + """ + Close the pool, and every connection still checked out of it. + + `psycopg_pool.close()` only closes the connections currently *idle in* the pool - one + that a task took and never gave back is not reachable from it, so closing the pool + leaves that socket open to the server. A task that ends without committing is exactly + that case, and one leaked connection per such task exhausts `max_connections` in a + long-running process (or partway through a test suite). + + These are closed outright rather than returned, because the pool they would go back to + is about to be closed anyway. Claiming the whole set in one statement, with no await in + between, keeps this atomic against a `_rollback_and_return` racing to claim the same + connection. + """ + checked_out, self._checked_out = list(self._checked_out), set() + + for conn in checked_out: + with contextlib.suppress(Exception): + await conn.close() + + await self._pool.close() + + +class SqliteAsyncConnection: + """ + Pool-like wrapper around the one connection used for `sqlite:memory`. + + Transactions remain open until `commit()` or `rollback()`. Because tasks cannot receive + separate transactions, a second task is refused while another owns one. `_lock` serializes + statements; `_owner` defines the transaction boundary. + """ + + def __init__(self, conn: AsyncConnection) -> None: + self._conn = conn + # created here rather than bound eagerly: asyncio.Lock() only attaches to a loop on + # first acquire, and this object is built inside `open_sqlite_async_connection()`. + self._lock = asyncio.Lock() + # the task whose transaction is currently open, if any. Not a ContextVar: the point is + # for *other* tasks to see it and be refused, which is the opposite of what a + # ContextVar's per-task isolation provides. + self._owner: "asyncio.Task[t.Any] | None" = None + # tasks with a reclaim callback armed, so one is armed per task rather than per + # statement. Entries are dropped when the callback fires. + self._reclaimable: "set[asyncio.Task[t.Any]]" = set() + + def _is_owned_elsewhere(self) -> bool: + """ + Whether the open transaction belongs to a task other than the calling one. + + No `done()` term, deliberately. On the `connection()` path the caller settles a + finished owner first; on the direct `commit()`/`rollback()` path a finished owner is + still somebody else's transaction and must therefore no-op. Treating a finished owner + as absent instead - which this used to do - is how a non-owner ended up committing or + rolling back another task's writes. + + `asyncio.current_task()` answers None off-task, and None is never stored as an owner, + so an off-task caller correctly reads any owner as somebody else's. + """ + return self._owner is not None and self._owner is not asyncio.current_task() + + def _abandoned_owner(self) -> "asyncio.Task[t.Any] | None": + """The finished task whose open transaction needs reclaiming, or None.""" + owner = self._owner + if owner is None or owner is asyncio.current_task() or not owner.done(): + return None + return owner + + async def _settle_abandoned_owner(self) -> None: + """ + Roll back a transaction whose owning task ended without committing it. + + Must be called with `_lock` held. `PostgresAsyncPool` and `SqliteAsyncPool` both hand + the abandoned connection back and roll it back on the way (`_reclaim`); there is no + connection to hand back here, so the transaction itself is what gets reclaimed. + + `settle_abandoned_sync()` normally gets there first, from the task's done-callback or + from `TypeDAL._settle_abandoned_async_writes()`. This is the deterministic backstop for + when neither has run yet, or when the sync path could not act because the lock was held. + + If the rollback fails, the owner is left set so the next task is refused rather than + allowed to inherit a transaction that could not be reclaimed. + """ + if self._abandoned_owner() is None: + return + + try: + await self._conn.rollback() + except Exception: # pragma: no cover - a hard rollback failure is not reachable through the public API + return + + self._owner = None + + def _refuse_if_owned_elsewhere(self) -> None: + """ + Refuse the caller if a different task holds the open transaction. + + Must be called with `_lock` held. Checking on the way *to* the lock instead lets a + second task read `_owner` while the first is still awaiting inside its `connection()` + block - before that block's `finally` has recorded the ownership - so it passes the + check, queues on the lock, and then walks straight into the transaction it should have + been refused from. Under the lock, the first task's ownership is always already visible. + """ + if self._is_owned_elsewhere(): + raise ConcurrentTransactionError( + "Another task holds an open transaction on this sqlite:memory database, and " + "SQLite cannot give the two of them separate ones - shared-cache mode refuses " + "a second concurrent writer. Commit or roll back that task before starting " + "here, or use a file-backed database, which does get a connection per task.", + ) + + def _take_ownership_if_in_transaction(self) -> None: + """ + Own the connection if the statement just run left a transaction open, else release it. + + Ownership tracks `in_transaction` rather than "used the connection at all", because + only a write opens a transaction here - sqlite3 implicitly BEGINs before DML and leaves + SELECT and DDL alone. Claiming on every use instead would mean a single `collect_async()` + locked every other task out of the database until the reader happened to commit, which + readers have no reason to do. + + Taking ownership also arms the done-callback that reclaims the transaction if this task + never ends it, the same safety net the two real pools arm at checkout. Armed here + rather than on entry to `connection()` because this is the moment there is something to + reclaim; `_reclaimable` keeps one callback per task rather than one per statement. + """ + owner = asyncio.current_task() if self._conn.in_transaction else None # ty: ignore[unresolved-attribute] + self._owner = owner + + if owner is not None and owner not in self._reclaimable: + self._reclaimable.add(owner) + owner.add_done_callback(self._reclaim) + + def settle_abandoned_sync(self) -> bool: + """ + Roll back a finished owner's transaction for synchronous callers. + + Return `False` when the connection is busy or rollback fails, so callers continue to + treat the async transaction as pending. + """ + if self._abandoned_owner() is None: + return True + + if self._lock.locked(): + return False + + try: + # aiosqlite's own `in_transaction` reaches through `_conn` the same way; this is + # the sqlite3 connection behind the queue, not the aiosqlite wrapper. + self._conn._conn.rollback() # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + except Exception: # pragma: no cover - a hard rollback failure is not reachable through the public API + return False + + self._owner = None + return True + + def _reclaim(self, task: "asyncio.Task[t.Any]") -> None: + """ + Roll back the transaction of a task that ended without committing it. + + Re-checks ownership rather than trusting the callback fired: by the time it runs the + task may have committed (so `_owner` is None), or another task may already hold the + transaction, and rolling *that* back is the very thing this class exists to prevent. + + A False from `settle_abandoned_sync()` is safe to ignore here: `_settle_abandoned_owner()` + runs on the next async `connection()`, so the still-open owner is refused rather than + inherited. + """ + self._reclaimable.discard(task) + + if self._owner is not task: + # the ordinary case - commit()/rollback() already ended it + return + + self.settle_abandoned_sync() + + @contextlib.asynccontextmanager + async def connection(self) -> t.AsyncIterator[AsyncConnection]: + async with self._lock: + await self._settle_abandoned_owner() + self._refuse_if_owned_elsewhere() + try: + yield self._conn + finally: + # in a finally: a statement that raised may still have opened the transaction, + # and leaving it unowned would let another task walk into it. + self._take_ownership_if_in_transaction() + + async def _end_transaction(self, end: t.Callable[[], t.Awaitable[None]]) -> None: + """ + End this task's transaction, or no-op when the transaction belongs elsewhere. + + Runs under the lock so a commit cannot land halfway through another coroutine's + `connection()` block and write out a statement it has not finished issuing. + + If another task owns the transaction, this does nothing. `PostgresAsyncPool` and + `SqliteAsyncPool` already no-op for a task that holds no connection; unlike them, the + connection here is shared, so acting anyway would commit or roll back writes the owner + has not finished issuing. The guard stays silent rather than raising because both + `commit_async()` and `rollback_async()` are what a caller reaches for while cleaning + up, often in a `finally`, where raising would mask whatever sent it there. + """ + async with self._lock: + if self._is_owned_elsewhere(): + return + + await end() + self._owner = None + + async def commit(self) -> None: + await self._end_transaction(self._conn.commit) + + async def rollback(self) -> None: + await self._end_transaction(self._conn.rollback) + + async def close(self) -> None: + await self._conn.close() + + +class SqliteAsyncPool: + """ + A connection per asyncio task for a file-backed SQLite database, giving it the same + per-task transaction boundary `PostgresAsyncPool` gives Postgres. + + Possible here and not for `sqlite:memory` because a file has a path two connections can + both open. WAL mode is what makes it worth doing - without it a writer blocks readers on a + database-wide lock and separate connections buy nothing. SQLite still permits exactly one + writer at a time, so two writing tasks serialize on `busy_timeout` rather than running + concurrently; that is a throughput limit, not a correctness one, and it surfaces as + `database is locked` if a task holds a write open longer than the timeout. + + Connections are opened per task rather than pooled and reused. SQLite connections are cheap + (no handshake, no network) so there is little to gain from recycling, and closing on + release keeps the file-handle count bounded by concurrent tasks rather than by peak usage. + """ + + def __init__(self, db: "TypeDAL") -> None: + self._db = db + self._current: contextvars.ContextVar[t.Any] = contextvars.ContextVar( + f"typedal_sqlite_conn_{id(self):x}", + default=None, + ) + # every connection handed out and not yet closed, so close() can reach the ones whose + # tasks ended without committing. Same reasoning as `PostgresAsyncPool._checked_out`. + self._open: set[t.Any] = set() + # see `_spawn_reclaim()` - without this the reclaim tasks can be collected mid-flight. + self._reclaim_tasks: "set[asyncio.Task[None]]" = set() + + def _own_connection(self) -> t.Any: + """ + The connection this task opened, or None - see `PostgresAsyncPool._own_connection` for + why an entry inherited from a parent task has to count as None. + """ + entry = self._current.get() + if entry is None: + return None + + owner, conn = entry + return conn if owner is asyncio.current_task() else None + + async def _acquire(self) -> t.Any: + if (conn := self._own_connection()) is not None: + return conn + + conn = await _connect_sqlite_async(self._db) + self._current.set((asyncio.current_task(), conn)) + self._open.add(conn) + + if task := asyncio.current_task(): + task.add_done_callback(lambda _task: self._reclaim(conn)) + + return conn + + def _reclaim(self, conn: t.Any) -> None: + """ + Close a connection whose task ended without committing or rolling back. + + Handed its connection directly rather than reading the `ContextVar`, because + `add_done_callback` runs in the loop's context and not the finished task's. + """ + if conn not in self._open: + return + + async def _rollback_and_close() -> None: + # claim by removing from `_open`, before the first await, so this is atomic against + # `_release` and `close()` - see `PostgresAsyncPool._reclaim`. Unlike there, the + # connection is closed rather than returned either way, so a lost claim only means + # somebody else already closed it. + if conn not in self._open: + return + + self._open.discard(conn) + + with contextlib.suppress(Exception): + await conn.rollback() + with contextlib.suppress(Exception): + await conn.close() + + _spawn_reclaim(_rollback_and_close(), self._reclaim_tasks) + + async def _release(self, conn: t.Any) -> None: + # handed the connection for the same reason as `PostgresAsyncPool._release`. + self._current.set(None) + self._open.discard(conn) + await conn.close() + + @contextlib.asynccontextmanager + async def connection(self) -> t.AsyncIterator[AsyncConnection]: + # no commit and no rollback on exit - the caller's transaction spans its calls and ends + # when it says so, exactly as on pydal's sync connection. + yield t.cast(AsyncConnection, await self._acquire()) + + def settle_abandoned_sync(self) -> bool: + """Per-task backends reclaim abandoned connections through their own done-callback.""" + return True + + async def commit(self) -> None: + if (conn := self._own_connection()) is None: + return + + await conn.commit() + await self._release(conn) + + async def rollback(self) -> None: + if (conn := self._own_connection()) is None: + return + + await conn.rollback() + await self._release(conn) + + async def close(self) -> None: + for conn in list(self._open): + self._open.discard(conn) + with contextlib.suppress(Exception): + await conn.rollback() + with contextlib.suppress(Exception): + await conn.close() + + +async def open_postgres_async_pool(db: "TypeDAL") -> AsyncConnectionPool: + """ + Async pool factory for Postgres (registered in `_ASYNC_POOL_FACTORIES`). + """ + try: + import psycopg_pool + except ImportError as e: # pragma: no cover + raise RuntimeError( + "The async execution path requires `psycopg[binary,pool]`. Install via `typedal[postgres-async]`.", + ) from e + + # pydal accepts 'postgres://', psycopg wants the standard 'postgresql://': + uri = db._uri.replace("postgres://", "postgresql://", 1) + # min_size=1 rather than psycopg_pool's default of 4: connections are held for the length + # of a task's transaction now, not one statement, but opening four server connections + # before anyone has asked for one is pure cost - it multiplies every short-lived `TypeDAL` + # by four against the server's max_connections. + # max_size must be passed explicitly: psycopg_pool defaults it to min_size, so min_size=1 + # alone would cap the pool at a single connection and deadlock the second concurrent task + # for the full 30s checkout timeout. + pool = psycopg_pool.AsyncConnectionPool(uri, min_size=1, max_size=POSTGRES_POOL_MAX_SIZE, open=False) + await pool.open() + return PostgresAsyncPool(pool) + + +def sqlite_is_in_memory(adapter: "SQLAdapter") -> bool: + """ + Whether pydal resolved this SQLite database to an in-memory one. + + Read off `dbpath` rather than the URI, because that is what pydal itself produced: for + `sqlite:memory` it builds `file:?mode=memory&cache=shared` and sets + `driver_args["uri"] = True` (adapters/sqlite.py), and it is the shared-cache part that + decides whether a second connection is possible at all. + """ + return "mode=memory" in str(adapter.dbpath) + + +async def _connect_sqlite_async(db: "TypeDAL") -> t.Any: + """ + One aiosqlite connection configured the way pydal configures its own. + """ + try: + import aiosqlite + except ImportError as e: # pragma: no cover + raise RuntimeError( + "The async execution path requires `aiosqlite`. Install with `pip install typedal[sqlite-async]`.", + ) from e + + adapter = db._adapter + # Reuse pydal's own path/URI resolution and connect kwargs (adapters/sqlite.py) - in + # particular the memory-mode shared-cache URI, so this connection sees the same in-memory + # database as pydal's own sync connection. + conn = await aiosqlite.connect(adapter.dbpath, **adapter.driver_args) + + # Mirror SQLite.after_connection() (adapters/sqlite.py): custom functions and PRAGMA + # are per-connection state, and this connection is not the one pydal set those up on. + await conn.create_function("web2py_extract", 2, adapter.web2py_extract) + await conn.create_function("REGEXP", 2, adapter.web2py_regexp) + if adapter.adapter_args.get("foreign_keys", True): + await conn.execute("PRAGMA foreign_keys=ON;") + + if not sqlite_is_in_memory(adapter): + # SQLite still allows a single writer, so two writing tasks queue here rather than + # failing outright. Genuinely per-connection, unlike journal_mode - see + # `enable_sqlite_wal()`. + await conn.execute(f"PRAGMA busy_timeout={SQLITE_BUSY_TIMEOUT_MS};") + + return conn + + +async def enable_sqlite_wal(db: "TypeDAL") -> None: + """ + Put a file-backed SQLite database into WAL mode, once. + + WAL is what lets one task read while another holds a write open; without it they serialize + on a database-wide lock and per-task connections buy nothing. + + Deliberately not part of `_connect_sqlite_async`. `journal_mode` is a persistent property + of the database *file*, not of a connection, so setting it per connection is both redundant + and actively harmful: switching into WAL needs an exclusive lock, and a second task opening + its connection while the first holds a write transaction gets `database is locked` for a + setting that was already applied. Done here instead, on its own connection, before the pool + exists and therefore before any task can be writing. + + Failure is tolerated. A database that cannot be switched (on a filesystem that does not + support WAL, say) still works through `SqliteAsyncPool` - tasks just contend more. + """ + conn = await _connect_sqlite_async(db) + try: + with contextlib.suppress(Exception): + await conn.execute("PRAGMA journal_mode=WAL;") + await conn.commit() + finally: + await conn.close() + + +async def open_sqlite_async_connection(db: "TypeDAL") -> AsyncConnectionPool: + """ + Async connection factory for SQLite (registered in `_ASYNC_POOL_FACTORIES`). + + Two shapes, because the two kinds of SQLite database genuinely differ. A file-backed one + supports a connection per task, so it gets `SqliteAsyncPool` and the same per-task + transaction boundary Postgres has. `sqlite:memory` does not - see + `ConcurrentTransactionError` - so it gets the single-connection `SqliteAsyncConnection`, + which refuses a second task rather than merging it into the first one's transaction. + """ + if sqlite_is_in_memory(db._adapter): + return SqliteAsyncConnection(await _connect_sqlite_async(db)) + + await enable_sqlite_wal(db) + return SqliteAsyncPool(db) + + +# A note the factories above share: `TypeDAL(..., after_connection=...)` does NOT run on these +# connections. That hook is pydal's, is handed the pydal adapter, and drives the sync cursor +# (connection.py) - there is no faithful way to replay it against a connection the adapter does +# not own, and a hook reaching into driver internals (`adapter.connection.create_function`) +# could not be replayed at all. What the factories do instead is mirror the backend's own +# `after_connection()` setup, above, so the async connection matches pydal's on everything +# pydal itself configures. pydal is not absolute about the hook either: a connection recycled +# from its global pool comes back with `run_hooks=False`. +# Covered by `test_after_connection_hook_does_not_reach_the_async_connection`. + +type PoolFactory = t.Callable[["TypeDAL"], t.Awaitable[AsyncConnectionPool]] + +ASYNC_POOL_FACTORIES: dict[str, PoolFactory] = { + "postgres": open_postgres_async_pool, + "sqlite": open_sqlite_async_connection, +} + + +class AsyncPoolManager: + """ + Owns the lazily-opened async connection for one `TypeDAL`: picking the factory for its + backend, keeping creation single, and closing/reopening. + + Its own object rather than three attributes and two methods on `TypeDAL`, because the + lifecycle has behaviour worth exercising on its own - "opened exactly once even when two + coroutines race for the first use", "an unknown backend fails loudly" - and `factories` as + a constructor argument makes that reachable directly, instead of only through a patched + module global. + """ + + def __init__(self, db: "TypeDAL", factories: dict[str, PoolFactory] | None = None) -> None: + self._db = db + self._factories = ASYNC_POOL_FACTORIES if factories is None else factories + self._pool: AsyncConnectionPool | None = None + self._lock: asyncio.Lock | None = None + self._lock_loop: asyncio.AbstractEventLoop | None = None + + @property + def pool(self) -> AsyncConnectionPool | None: + """ + The connection if one is currently open, else None. Never opens one - use `get()`. + """ + return self._pool + + def _get_lock(self) -> asyncio.Lock: + """ + The lock guarding creation, bound to the loop currently running. + + Not created once in `__init__`: an `asyncio.Lock` binds to the loop it is first used on + and refuses use from another one, while a `TypeDAL` can outlive a loop (every + pytest-asyncio test gets a fresh one, and `close()` explicitly supports reopening). + Re-created when the loop changed - safe to decide here because this method never + awaits, so two coroutines on one loop cannot interleave inside it and always come away + with the same lock. + """ + loop = asyncio.get_running_loop() + if self._lock is None or self._lock_loop is not loop: + self._lock = asyncio.Lock() + self._lock_loop = loop + + return self._lock + + async def get(self) -> AsyncConnectionPool: + """ + The async connection for this db, opening it on first use. + + Creation happens under the lock with the check repeated inside it: the factories await, + so a plain `if self._pool is None: self._pool = await factory(...)` lets two coroutines + whose first use overlaps both pass the check and both open one. Only one could be + stored, and the other would be dropped without `close()` - a leaked pool, or on SQLite + a leaked connection and its background thread. + """ + if self._pool is not None: + # fast path: already open, no need to take the lock at all + return self._pool + + async with self._get_lock(): + if self._pool is None: + dbengine = self._db._adapter.dbengine + try: + factory = self._factories[dbengine] + except KeyError: + raise NotImplementedError( + f"The async execution path is only implemented for " + f"{', '.join(self._factories)}, not {dbengine!r}.", + ) from None + + self._pool = await factory(self._db) + + return self._pool + + async def close(self) -> None: + """ + Close the connection if one was ever opened, leaving this manager reusable. + """ + if self._pool is not None: + await self._pool.close() + self._pool = None + + +async def postgres_lastrowid_async( + adapter: SQLAdapter, + table: pydal.objects.Table, + cursor: AsyncCursor, + last_insert: LastInsert, +) -> int | None: + """ + Async twin of `Postgre.lastrowid()` (pydal adapters/postgres.py). + + `last_insert` is the value `adapter._insert()` set as a side effect of building the INSERT + statement (postgres.py, set whenever the table has a standard `_id` column), passed + in by `insert_async()` rather than read back off the adapter here. It has to be passed: + `adapter._last_insert` is a property over `THREAD_LOCAL._pydal_last_insert_` + (postgres.py), and every coroutine on this path shares one thread, so reading it + after the intervening awaits would see whichever insert touched it last. + + Truthy means the id is already in the RETURNING result of the statement just executed, read + here with a plain `fetchone()`, no extra round trip. Otherwise (a custom `_primarykey` not + covered by RETURNING, or a `DEFAULT VALUES` insert) fall back to `currval()`, a real second + query - on this same connection, so it sees this insert's sequence value. + """ + if last_insert: + row = await cursor.fetchone() + return int(row[0]) + + sequence_name = table._sequence_name + await cursor.execute("SELECT currval(%s);" % adapter.adapt(sequence_name)) + row = await cursor.fetchone() + return int(row[0]) + + +async def sqlite_lastrowid_async( + _adapter: SQLAdapter, + _table: pydal.objects.Table, + cursor: AsyncCursor, + _last_insert: LastInsert, +) -> int | None: + """ + Async twin of the base `SQLAdapter.lastrowid()` (pydal adapters/base.py), used by + SQLite (no override there). `cursor.lastrowid` is a plain attribute, not awaitable, and + needs no `last_insert` - it takes the argument only to share one strategy signature. + """ + return cursor.lastrowid + + +# One lastrowid strategy per backend, mirroring `ASYNC_POOL_FACTORIES` - `insert_async()` looks +# this up by `adapter.dbengine` rather than branching, same reasoning as the pool factories above. +LASTROWID_STRATEGIES: dict[ + str, + t.Callable[[SQLAdapter, pydal.objects.Table, AsyncCursor, LastInsert], t.Awaitable[int | None]], +] = { + "postgres": postgres_lastrowid_async, + "sqlite": sqlite_lastrowid_async, +} + + +# Postgres command tags that report a statement which changed nothing. Everything else either +# modified data or changed schema, and therefore left work the sync connection cannot see. +# Inverted like this on purpose: an unrecognised tag then counts as a write, which costs a held +# connection at worst, where the other way round costs a silently discarded statement. +POSTGRES_READ_ONLY_COMMAND_TAGS = frozenset( + {"SELECT", "SHOW", "EXPLAIN", "FETCH", "MOVE", "CLOSE", "SET", "RESET", "BEGIN", "COMMIT", "ROLLBACK"}, +) + + +def postgres_left_uncommitted_work(_conn: AsyncConnection, cur: AsyncCursor) -> bool: + """ + Whether the statement this psycopg cursor just ran left uncommitted work. + + Read off the command tag the *server* sent back (`INSERT 0 1`, `UPDATE 3`, `CREATE TABLE`, + `SELECT 5`), not off the SQL that was sent. That is what makes this reliable where a text + prefix is not: a CTE-wrapped `INSERT`, a statement behind a leading comment and DDL all + report their real command here. + + `conn.info.transaction_status` cannot answer this on Postgres - psycopg opens a transaction + for a plain `SELECT` too, so it reports `INTRANS` for statements with nothing to commit. + """ + tag = str(getattr(cur, "statusmessage", "") or "").split(" ", 1)[0].upper() + return tag not in POSTGRES_READ_ONLY_COMMAND_TAGS + + +def sqlite_left_uncommitted_work(conn: AsyncConnection, _cur: AsyncCursor) -> bool: + """ + Whether the statement this aiosqlite connection just ran left uncommitted work. + + sqlite3 implicitly BEGINs before DML and leaves SELECT and DDL in autocommit, so + `in_transaction` *is* the question being asked - no command tag needed (and none exists). + + That DDL is excluded is the driver's behaviour, not a choice made here: a SQLite + `CREATE TABLE` is durable the moment it runs, so there is nothing pending to report. + """ + # not on the `AsyncConnection` protocol: psycopg has no counterpart, this is aiosqlite's. + return bool(conn.in_transaction) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + + +# One "did this leave uncommitted work" strategy per backend, mirroring `ASYNC_POOL_FACTORIES`. +# Used by `executesql_async()`, which is the one `_async` method handed arbitrary SQL and so the +# only one that cannot know up front whether it is about to write. +UNCOMMITTED_WORK_STRATEGIES: dict[str, t.Callable[[AsyncConnection, AsyncCursor], bool]] = { + "postgres": postgres_left_uncommitted_work, + "sqlite": sqlite_left_uncommitted_work, +} + + +async def base_delete_async(db: "TypeDAL", table: pydal.objects.Table, query: pydal.objects.Query) -> int | None: + """ + Async twin of the base `SQLAdapter.delete()` (pydal adapters/base.py): plain + build/execute sandwich, no cascade handling. Used directly for Postgres (no override + there), and internally by `sqlite_delete_async` for the actual delete statement - + mirroring how `SQLite.delete()` itself calls `super().delete()` for that part. + """ + adapter = db._adapter + sql = adapter._delete(table, query) + + pool = await db._get_async_pool() + with db._mark_async_pending(): + async with pool.connection() as conn, conn.cursor() as cur: + await cur.execute(sql) + try: + return cur.rowcount + except Exception: # pragma: no cover + # defensive, mirroring `adapter.delete()` (adapters/base.py): + # neither driver's `rowcount` actually raises, it is a plain property. + return None + + +async def sqlite_delete_async(db: "TypeDAL", table: pydal.objects.Table, query: pydal.objects.Query) -> int | None: + """ + Async twin of `SQLite.delete()` (pydal adapters/sqlite.py) - NOT a plain sandwich: + selects affected ids first, deletes, then recurses per cascaded FK with + `ondelete=CASCADE`. Recursion goes through `db.delete_async()` again (not this function + directly), so a cascaded delete on another table gets the dbengine-appropriate treatment + too, same as the original. + """ + # `_hold_connection`: the delete below (and the cascades after it) have to share this + # snapshot's transaction, or the rows cascaded to are chosen from ids read outside it. + id_rows = await db.select_async(query, table._id, _hold_connection=True) + deleted = [row[table._id.name] for row in id_rows] + + counter = await base_delete_async(db, table, query) + + if counter: + for field in table._referenced_by: + if field.type == "reference " + table._dalname and field.ondelete == "CASCADE": + cascade_query = field.belongs(deleted) + cascade_table = db._adapter.get_table(cascade_query) + await db.delete_async(cascade_table, cascade_query) + + return counter + + +# One delete strategy per backend, same reasoning as `ASYNC_POOL_FACTORIES`/`LASTROWID_STRATEGIES` +# - SQLite's isn't a plain sandwich (see `sqlite_delete_async`), Postgres's is. +DELETE_STRATEGIES: dict[ + str, + t.Callable[["TypeDAL", pydal.objects.Table, pydal.objects.Query], t.Awaitable[int | None]], +] = { + "postgres": base_delete_async, + "sqlite": sqlite_delete_async, +} diff --git a/src/typedal/caching.py b/src/typedal/caching.py index 9d01b4e..fd31170 100644 --- a/src/typedal/caching.py +++ b/src/typedal/caching.py @@ -22,6 +22,13 @@ from .query_builder import QueryBuilder +class FunctionWithMetadata[T](t.Protocol): + __name__: str + __qualname__: str + + def __call__(self, *args: t.Any, **kwargs: t.Any) -> T: ... + + def get_now(tz: dt.timezone = dt.timezone.utc) -> dt.datetime: """ Get the default datetime, optionally in a specific timezone. @@ -306,7 +313,8 @@ def _fetch_cached_payload(key: str) -> tuple[t.Any, t.Any] | None: now = get_now() # Ensure comparison is offset-aware if the row has a timestamp - expires = row.expires_at.replace(tzinfo=dt.timezone.utc) if row.expires_at else None + expires_at = t.cast(dt.datetime | None, row.expires_at) + expires = expires_at.replace(tzinfo=dt.timezone.utc) if expires_at else None if expires and now >= expires: row.delete_record() @@ -531,7 +539,7 @@ def calculate_stats(db: "TypeDAL") -> Stats[GenericStats]: def memoize[T: t.Any]( db: "TypeDAL", - func: t.Callable[..., T], + func: FunctionWithMetadata[T], *args: TypedRows[t.Any] | TypedTable, key: str | None = None, ttl: int | dt.timedelta | dt.datetime | None = None, @@ -568,7 +576,7 @@ def memoize[T: t.Any]( for row in arg: deps.add((str(row._table), row.id)) elif isinstance(arg, TypedTable): - deps.add((str(arg._table), arg.id)) + deps.add((str(arg._table), t.cast(int, arg.id))) # Generate cache key _, hashed_key = create_and_hash_cache_key(key, *[getattr(arg, "id", None) for arg in args], kwargs) diff --git a/src/typedal/cli.py b/src/typedal/cli.py index 5da52f7..20519dc 100644 --- a/src/typedal/cli.py +++ b/src/typedal/cli.py @@ -3,10 +3,9 @@ """ import sys -import typing +import typing as t import warnings from pathlib import Path -from typing import Optional import tomli from configuraptor import asdict @@ -34,7 +33,6 @@ ) exit(127) # command not found -from typing import Never from pydal2sql.typer_support import IS_DEBUG, with_exit_code from pydal2sql.types import ( @@ -59,12 +57,12 @@ no_args_is_help=True, ) -questionary_types: dict[typing.Hashable, Optional[AnyDict]] = { +questionary_types: dict[t.Hashable, AnyDict | None] = { str: { "type": "text", "validate": lambda text: True if len(text) > 0 else "Please enter a value", }, - Optional[str]: { + str | None: { "type": "text", # no validate because it's optional }, @@ -109,11 +107,11 @@ notfound = object() -def _get_question[T](prop: str, annotation: typing.Type[T]) -> Optional[AnyDict]: # pragma: no cover +def _get_question[T](prop: str, annotation: t.Type[T]) -> AnyDict | None: # pragma: no cover question = questionary_types.get(prop, notfound) if question is notfound: # None means skip the question, notfound means use the type default! - question = questionary_types.get(annotation) # type: ignore + question = questionary_types.get(annotation) if not question: return None @@ -121,7 +119,7 @@ def _get_question[T](prop: str, annotation: typing.Type[T]) -> Optional[AnyDict] return question.copy() # type: ignore -def get_question[T](prop: str, annotation: typing.Type[T], default: T | None) -> Optional[T]: # pragma: no cover +def get_question[T](prop: str, annotation: t.Type[T], default: T | None) -> T | None: # pragma: no cover """ Generate a question based on a config property and prompt the user for it. """ @@ -130,19 +128,19 @@ def get_question[T](prop: str, annotation: typing.Type[T], default: T | None) -> question["name"] = prop question["message"] = question.get("message", f"{prop}? ") - default = typing.cast(T, default or question.get("default") or "") + default = t.cast(T, default or question.get("default") or "") if annotation is int: - default = typing.cast(T, str(default)) + default = t.cast(T, str(default)) response = questionary.unsafe_prompt([question], default=default)[prop] - return typing.cast(T, response) + return t.cast(T, response) @app.command() @with_exit_code(hide_tb=IS_DEBUG) def setup( - config_file: typing.Annotated[Optional[str], typer.Option("--config", "-c")] = None, + config_file: t.Annotated[str | None, typer.Option("--config", "-c")] = None, minimal: bool = False, ) -> None: # pragma: no cover """ @@ -202,7 +200,7 @@ def setup( _fill_defaults(data, prop, data.get(prop)) default_value = data.get(prop, None) - answer: typing.Any = get_question(prop, annotation, default_value) + answer: t.Any = get_question(prop, annotation, default_value) if isinstance(answer, str): answer = answer.strip() @@ -212,7 +210,7 @@ def setup( elif annotation is int: answer = int(answer) - config.update(**{prop: answer}) + config.update(**{prop: t.cast(t.Any, answer)}) data[prop] = answer for prop in TypeDALConfig.__annotations__: @@ -239,16 +237,16 @@ def setup( @app.command(name="migrations.generate") @with_exit_code(hide_tb=IS_DEBUG) def generate_migrations( - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, filename_before: OptionalArgument[str] = None, filename_after: OptionalArgument[str] = None, dialect: DBType_Option = None, tables: Tables_Option = None, - magic: Optional[bool] = None, - noop: Optional[bool] = None, - function: Optional[str] = None, + magic: bool | None = None, + noop: bool | None = None, + function: str | None = None, output_format: OutputFormat_Option = None, - output_file: Optional[str] = None, + output_file: str | None = None, dry_run: bool = False, ) -> bool: # pragma: no cover """ @@ -310,18 +308,18 @@ def generate_migrations( @app.command(name="migrations.run") @with_exit_code(hide_tb=IS_DEBUG) def run_migrations( - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, migrations_file: OptionalArgument[str] = None, - db_uri: Optional[str] = None, - db_folder: Optional[str] = None, - schema_version: Optional[str] = None, - redis_host: Optional[str] = None, - migrate_cat_command: Optional[str] = None, - database_to_restore: Optional[str] = None, - migrate_table: Optional[str] = None, - flag_location: Optional[str] = None, - schema: Optional[str] = None, - create_flag_location: Optional[bool] = None, + db_uri: str | None = None, + db_folder: str | None = None, + schema_version: str | None = None, + redis_host: str | None = None, + migrate_cat_command: str | None = None, + database_to_restore: str | None = None, + migrate_table: str | None = None, + flag_location: str | None = None, + schema: str | None = None, + create_flag_location: bool | None = None, dry_run: bool = False, ) -> bool: # pragma: no cover """ @@ -368,13 +366,13 @@ def run_migrations( @app.command(name="migrations.fake") @with_exit_code(hide_tb=IS_DEBUG) def fake_migrations( - names: typing.Annotated[list[str], typer.Argument()] = None, + names: t.Annotated[list[str] | None, typer.Argument()] = None, all: bool = False, # noqa: A002 - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, - migrations_file: Optional[str] = None, - db_uri: Optional[str] = None, - db_folder: Optional[str] = None, - migrate_table: Optional[str] = None, + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, + migrations_file: str | None = None, + db_uri: str | None = None, + db_folder: str | None = None, + migrate_table: str | None = None, dry_run: bool = False, ) -> int: # pragma: no cover """ @@ -452,12 +450,12 @@ def fake_migrations( @app.command(name="migrations.stub") @with_exit_code(hide_tb=IS_DEBUG) def migrations_stub( - migration_name: typing.Annotated[str, typer.Argument()] = "stub_migration", - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, + migration_name: t.Annotated[str, typer.Argument()] = "stub_migration", + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, output_format: OutputFormat_Option = None, - output_file: Optional[str] = None, - dry_run: typing.Annotated[bool, typer.Option("--dry", "--dry-run")] = False, - is_pydal: typing.Annotated[bool, typer.Option("--pydal", "-p")] = False, + output_file: str | None = None, + dry_run: t.Annotated[bool, typer.Option("--dry", "--dry-run")] = False, + is_pydal: t.Annotated[bool, typer.Option("--pydal", "-p")] = False, # defaults to is_typedal of course ) -> int: """ @@ -484,12 +482,12 @@ def migrations_stub( @app.command(name="typescript.generate") @with_exit_code(hide_tb=IS_DEBUG) def generate_typescript( - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, filename: OptionalArgument[str] = None, tables: Tables_Option = None, - magic: Optional[bool] = None, - function: Optional[str] = None, - output_file: Optional[str] = None, + magic: bool | None = None, + function: str | None = None, + output_file: str | None = None, ) -> bool: """ Generate TypeScript interfaces from TypeDAL table definitions. @@ -563,10 +561,10 @@ def tabulate_data(data: AnyNestedDict) -> None: print(tabulate(flattened_data, headers="keys")) -type FormatOptions = typing.Literal["plaintext", "json", "yaml", "toml"] +type FormatOptions = t.Literal["plaintext", "json", "yaml", "toml"] -def get_output_format(fmt: FormatOptions) -> typing.Callable[[AnyNestedDict], None]: +def get_output_format(fmt: FormatOptions) -> t.Callable[[AnyNestedDict], None]: """ This function takes a format option as input and \ returns a function that can be used to output data in the specified format. @@ -596,7 +594,7 @@ def output(_data: AnyDict | AnyNestedDict) -> None: print(tomli_w.dumps(_data)) case _: - options = typing.get_args(FormatOptions) + options = t.get_args(FormatOptions) raise ValueError(f"Invalid format '{fmt}'. Please choose one of {options}.") return output @@ -605,11 +603,9 @@ def output(_data: AnyDict | AnyNestedDict) -> None: @app.command(name="cache.stats") @with_exit_code(hide_tb=IS_DEBUG) def cache_stats( - identifier: typing.Annotated[str, typer.Argument()] = "", - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, - fmt: typing.Annotated[ - str, typer.Option("--format", "--fmt", "-f", help="plaintext (default) or json") - ] = "plaintext", + identifier: t.Annotated[str, typer.Argument()] = "", + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, + fmt: t.Annotated[str, typer.Option("--format", "--fmt", "-f", help="plaintext (default) or json")] = "plaintext", ) -> None: # pragma: no cover """ Collect caching stats. @@ -622,7 +618,7 @@ def cache_stats( config = load_config(connection) db = TypeDAL(config=config, migrate=False, fake_migrate=False) - output = get_output_format(typing.cast(FormatOptions, fmt)) + output = get_output_format(t.cast(FormatOptions, fmt)) data: AnyDict parts = identifier.split(".") @@ -651,8 +647,8 @@ def cache_stats( @app.command(name="cache.clear") @with_exit_code(hide_tb=IS_DEBUG) def cache_clear( - connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None, - purge: typing.Annotated[bool, typer.Option("--all", "--purge", "-p")] = False, + connection: t.Annotated[str | None, typer.Option("--connection", "-c")] = None, + purge: t.Annotated[bool, typer.Option("--all", "--purge", "-p")] = False, ) -> None: # pragma: no cover """ Clear (expired) items from the cache. @@ -674,7 +670,7 @@ def cache_clear( db.commit() -def version_callback() -> Never: +def version_callback() -> t.Never: """ --version requested! """ @@ -683,7 +679,7 @@ def version_callback() -> Never: raise typer.Exit(0) -def config_callback() -> Never: +def config_callback() -> t.Never: """ --show-config requested. """ diff --git a/src/typedal/core.py b/src/typedal/core.py index 10a5991..dfb5e74 100644 --- a/src/typedal/core.py +++ b/src/typedal/core.py @@ -5,15 +5,27 @@ from __future__ import annotations # noinspection PyUnusedImports +import asyncio +import collections +import contextlib import datetime as dt import sys import typing as t import warnings from pathlib import Path -from typing import Optional import pydal +from .async_execution import ( + DELETE_STRATEGIES, + LASTROWID_STRATEGIES, + UNCOMMITTED_WORK_STRATEGIES, + AsyncConnectionPool, + AsyncPoolManager, + ConcurrentTransactionError, + SyncTransactionTracker, + TransactionSplitError, +) from .config import LazyPolicy, TypeDALConfig, load_config from .helpers import ( SYSTEM_SUPPORTS_TEMPLATES, @@ -32,7 +44,7 @@ from annotationlib import ForwardRef except ImportError: # pragma: no cover # python 3.13- - from typing import ForwardRef + from typing import ForwardRef # special case, keep `from typing` if t.TYPE_CHECKING: from .fields import TypedField @@ -40,6 +52,36 @@ from .types import AnyDict, DefineKwargs, Expression, Rows, Set, T_Query, Table +# stands in for the task in `TypeDAL._async_pending_owners` when there isn't one. Its own +# object rather than None so it cannot collide with a real entry, and so the "is this owner +# still running?" test can special-case it explicitly instead of by falsiness. +NO_ASYNC_TASK = object() + + +class _AsyncPendingMark: + """ + Handle on one `TypeDAL._mark_async_pending()` block, so it can be withdrawn again. + + Exists for `executesql_async()`, which is handed arbitrary SQL: it has to mark before + running the statement (a later mark would leave a window for a sync write to slip in), but + only afterwards can it ask the connection whether anything was actually left uncommitted. + """ + + def __init__(self, db: "TypeDAL", owner: t.Any, already_pending: bool) -> None: + self._db = db + self._owner = owner + # a mark this block did not add is not this block's to withdraw: an earlier `_async` + # write in the same task still holds its transaction open. + self._already_pending = already_pending + + def release(self) -> None: + """ + Withdraw this mark, if it was this block that placed it. + """ + if not self._already_pending: + self._db._async_pending_owners.discard(self._owner) + + def _expression_subclasses() -> t.Iterator[type]: """ Yield pydal.objects.Expression and every (nested) subclass currently loaded, e.g. Field and TypedField. @@ -99,7 +141,7 @@ def evaluate_forward_reference_312(fw_ref: ForwardRef, namespace: dict[str, type """ return t.cast( type, - fw_ref._evaluate( + fw_ref._evaluate( # ty: ignore[deprecated] localns=locals(), globalns=globals() | namespace, recursive_guard=frozenset(), @@ -115,7 +157,7 @@ def evaluate_forward_reference_313(fw_ref: ForwardRef, namespace: dict[str, type """ return t.cast( type, - fw_ref._evaluate( + fw_ref._evaluate( # ty: ignore[deprecated] localns=locals(), globalns=globals() | namespace, recursive_guard=frozenset(), @@ -163,7 +205,7 @@ def resolve_annotation_313(ftype: str, namespace: dict[str, type] | None = None) Variant for Python 3.13 """ - fw_ref: ForwardRef = t.get_args(t.Type[ftype])[0] + fw_ref: ForwardRef = t.get_args(t.Type[ftype])[0] # ty: ignore[invalid-type-form] return evaluate_forward_reference(fw_ref, namespace=namespace) @@ -229,6 +271,22 @@ class TypeDAL(_TypeDALBase): _config: TypeDALConfig _builder: TableDefinitionBuilder + # appended to, not replaced: pydal's own TimingHandler is what fills `db._timings`, and + # dropping it would take that with it. + execution_handlers = [*pydal.DAL.execution_handlers, SyncTransactionTracker] # noqa: RUF012 + + # whether each of the two connections holds an open transaction. See `TransactionSplitError` + # for why the pair has to be tracked at all. + # + # The two are shaped differently on purpose. pydal's sync connection really is one shared + # thing - `THREAD_LOCAL` gives one per thread, and every coroutine on an event loop is the + # same thread - so a single flag describes it exactly. The async side keeps a connection + # *per task* (`PostgresAsyncPool`), so "has uncommitted writes" is a per-task fact and a + # single flag cannot hold it: one task's `commit_async()` would clear it on behalf of every + # other task, and the guard would then wave through exactly the read it exists to refuse. + _sync_pending: bool + _async_pending_owners: set[t.Any] + # similar to the insert/update/delete hooks at table-level but for .collect/.execute: # note: return values are ignored! _before_collect: list[t.Callable[["QueryBuilder[t.Any]"], None]] @@ -238,34 +296,34 @@ class TypeDAL(_TypeDALBase): def __init__( self, - uri: Optional[str] = None, # default from config or 'sqlite:memory' - pool_size: int = None, # default 1 if sqlite else 3 - folder: Optional[str | Path] = None, # default 'databases' in config + uri: str | None = None, # default from config or 'sqlite:memory' + pool_size: int | None = None, # default 1 if sqlite else 3 + folder: str | Path | None = None, # default 'databases' in config db_codec: str = "UTF-8", - check_reserved: Optional[list[str]] = None, - migrate: Optional[bool] = None, # default True by config - fake_migrate: Optional[bool] = None, # default False by config + check_reserved: list[str] | None = None, + migrate: bool | None = None, # default True by config + fake_migrate: bool | None = None, # default False by config migrate_enabled: bool = True, fake_migrate_all: bool = False, decode_credentials: bool = False, - driver_args: Optional[AnyDict] = None, - adapter_args: Optional[AnyDict] = None, + driver_args: AnyDict | None = None, + adapter_args: AnyDict | None = None, attempts: int = 5, auto_import: bool = False, bigint_id: bool = False, debug: bool = False, lazy_tables: bool = False, - db_uid: Optional[str] = None, - after_connection: t.Callable[..., t.Any] = None, - tables: Optional[list[str]] = None, + db_uid: str | None = None, + after_connection: t.Callable[..., t.Any] | None = None, + tables: list[str] | None = None, ignore_field_case: bool = True, entity_quoting: bool = True, - table_hash: Optional[str] = None, - enable_typedal_caching: bool = None, + table_hash: str | None = None, + enable_typedal_caching: bool | None = None, use_pyproject: bool | str = True, use_env: bool | str = True, - connection: Optional[str] = None, - config: Optional[TypeDALConfig] = None, + connection: str | None = None, + config: TypeDALConfig | None = None, lazy_policy: LazyPolicy | None = None, ) -> None: """ @@ -293,6 +351,12 @@ def __init__( self._after_collect = [] self._before_execute = [] self._after_execute = [] + self._async_pools = AsyncPoolManager(self) # lazily-opened async connection; see _get_async_pool + + # set before super().__init__(), which migrates and therefore already executes + # statements through SyncTransactionTracker. + self._sync_pending = False + self._async_pending_owners = set() if config.folder: Path(config.folder).mkdir(exist_ok=True) @@ -327,11 +391,29 @@ def __init__( self.try_define(_TypedalCache) self.try_define(_TypedalCacheDependency) + def commit(self) -> None: + """ + Commit the transaction on pydal's own (synchronous) connection. + + Says nothing about the async connection - that one is ended by `commit_async()`. What it + does do is clear the flag that blocks the async path, so committing here is how you make + the other side usable again after a sync write. + """ + super().commit() + self._sync_pending = False + + def rollback(self) -> None: + """ + Roll back the transaction on pydal's own (synchronous) connection. See `commit`. + """ + super().rollback() + self._sync_pending = False + def close(self) -> None: """Close the database connection and unbind all defined TypedTable models.""" adapter = self._adapter try: - super().close() + super().close() # ty: ignore[unresolved-attribute] finally: for model in set(self._builder.class_map.values()): model.unbind() @@ -458,7 +540,7 @@ def wrapper(cls: t.Type[T]) -> t.Type[T]: return wrapper - def __call__(self, *_args: T_Query, **kwargs: t.Any) -> "TypedSet": + def __call__(self, *_args: T_Query, **kwargs: t.Any) -> "TypedSet": # ty: ignore[invalid-method-override] """ A db instance can be called directly to perform a query. @@ -491,7 +573,7 @@ def __getitem__(self, key: str) -> "Table": Example: db['users'] -> user """ - return t.cast(Table, super().__getitem__(str(key))) + return t.cast(Table, super().__getitem__(str(key))) # ty: ignore[unresolved-attribute] def find_model(self, table_name: str) -> t.Type["TypedTable"] | None: """ @@ -535,10 +617,10 @@ def executesql( query: str | Template, placeholders: t.Iterable[str] | dict[str, str] | None = None, as_dict: bool = False, - fields: t.Iterable[Field | TypedField[t.Any]] | None = None, + fields: "Field | TypedField[t.Any] | Table | t.Iterable[Field | TypedField[t.Any]] | None" = None, colnames: t.Iterable[str] | None = None, as_ordered_dict: bool = False, - ) -> list[t.Any]: + ) -> list[t.Any] | None: """ Executes a raw SQL statement or a TypeDAL template query. @@ -571,7 +653,7 @@ def executesql( if SYSTEM_SUPPORTS_TEMPLATES and isinstance(query, Template): # pragma: no cover query = sql_escape_template(self, query) - rows: list[t.Any] = super().executesql( + rows: list[t.Any] = super().executesql( # ty: ignore[unresolved-attribute] query, placeholders=placeholders, as_dict=as_dict, @@ -582,6 +664,490 @@ def executesql( return rows + # ------------------------------------------------------------------ + # Async execution path. + # ------------------------------------------------------------------ + + async def _get_async_pool(self) -> AsyncConnectionPool: + """ + The async connection (a real pool for Postgres, a single wrapped connection for SQLite) + for this instance, opened on first use. + + Necessarily a separate connection from pydal's own thread-local sync connection - pydal + drives Postgres with psycopg2 and SQLite with sqlite3, neither of which can be awaited - + and therefore a separate transaction. Rather than let a read silently miss the other + side's uncommitted work, this refuses to run while the sync side has any; the reasoning + is in `TransactionSplitError`. + + The lifecycle itself lives in `AsyncPoolManager` (async_execution.py). + """ + if self._sync_pending: + raise TransactionSplitError( + "The synchronous connection has uncommitted writes, which this async statement " + "would not see. Call `db.commit()` or `db.rollback()` first.", + ) + + return await self._async_pools.get() + + def _async_pending_owner(self) -> t.Any: + """ + The key the calling task's pending async writes are recorded under. + + The task itself where there is one. `asyncio.current_task()` answers None for a + coroutine driven without one, which is rare but not impossible; `NO_ASYNC_TASK` keeps + those recorded rather than silently untracked, at the cost of only being cleared by an + explicit `commit_async()`/`rollback_async()` - there is no task whose end could stand + in for that. + """ + return asyncio.current_task() or NO_ASYNC_TASK + + @contextlib.contextmanager + def _mark_async_pending(self) -> t.Iterator["_AsyncPendingMark"]: + """ + Mark this task as holding an async write for the duration of the write. + + Entered before connection acquisition so a synchronous write from another coroutine + cannot slip in during the await. Only `ConcurrentTransactionError` un-marks because + that caller was refused before opening a transaction; any other failure may have left + one open. + + Yields a handle whose `release()` withdraws the mark, for the one caller that has to + mark before it can know whether there was anything to mark - see `executesql_async`. + """ + owner = self._async_pending_owner() + # an earlier `_async` write in this same task already marked it, and its transaction is + # still open - this block's failure says nothing about that one, so leave it recorded. + already_pending = owner in self._async_pending_owners + + self._async_pending_owners.add(owner) + try: + yield _AsyncPendingMark(self, owner, already_pending) + except ConcurrentTransactionError: + if not already_pending: + self._async_pending_owners.discard(owner) + raise + + async def _release_readonly_connection(self, pool: AsyncConnectionPool) -> None: + """ + Release a connection used only for reading, unless this task has pending writes. + + Rollback is the pool-level release operation. Failures are suppressed because this is + `finally` cleanup and the task callback can still reclaim a Postgres connection. + """ + if self._async_pending_owner() in self._async_pending_owners: + return + + with contextlib.suppress(Exception): + await pool.rollback() + + async def _release_held_connection(self) -> None: + """ + Hand back a connection a `select_async(..., _hold_connection=True)` is still holding. + + For the callers that hold one for a write which then turns out not to happen - a + `_before_update`/`_before_delete` hook cancelling it, or nothing to update. Without this + the read's connection stays checked out (on Postgres: idle in transaction) until the + task ends, for a transaction that will never receive its write. + + Goes to `AsyncPoolManager.pool` rather than `_get_async_pool()`: if no pool was ever + opened there is nothing held, and releasing must not be what opens one. + """ + if pool := self._async_pools.pool: + await self._release_readonly_connection(pool) + + def _settle_abandoned_async_writes(self) -> bool: + """ + Reclaim an abandoned `sqlite:memory` async transaction, if there is one. + + Only the sync side needs this: pydal's `ExecutionHandler` cannot await the async pool, + and `sqlite:memory` has no connection to hand back. The two per-task backends already + reclaim their abandoned connections through their own done-callbacks. + + Returns False when the reclaim could not be completed. The caller must then treat the + async side as still pending, so the sync statement raises `TransactionSplitError` + instead of walking into a driver lock. + """ + pool = self._async_pools.pool + return pool is None or pool.settle_abandoned_sync() + + def _has_pending_async_writes(self) -> bool: + """ + Whether any task holds uncommitted async writes. + + Reclaim a finished `sqlite:memory` owner before checking. If reclaim fails, retain the + pending state; otherwise discard finished tasks whose abandoned writes were rolled back. + """ + if not self._settle_abandoned_async_writes(): + return True + + self._async_pending_owners = { + owner for owner in self._async_pending_owners if owner is NO_ASYNC_TASK or not owner.done() + } + + return bool(self._async_pending_owners) + + async def select_async( + self, + query: pydal.objects.Query, + *fields: t.Any, + _hold_connection: bool = False, + **attributes: t.Any, + ) -> pydal.objects.Rows: + """ + Async twin of `db(query).select(*fields, **attributes)`. + + Mirrors `Set.select()` (pydal objects.py) and `SQLAdapter.select()`/ + `_select_aux()` (adapters/base.py): build via pydal's own + `tables()`/`expand_all()`/`_select_wcols()` (pure, no I/O), execute via the async + driver for this backend (the only I/O, on our own connection, not pydal's; see + `ASYNC_POOL_FACTORIES`), parse via pydal's own `parse()` (pure). + + `_hold_connection` keeps this task's connection instead of handing it back afterwards, + so a write issued next lands in the same transaction as this read. Internal, and for + exactly one situation: a write that first has to know *which* rows it is about to touch + (`QueryBuilder.update_async`/`delete_async`, `sqlite_delete_async`). Without it those + two statements run in separate transactions and the ids reported back can describe rows + the write never touched - where the synchronous path, sharing one connection throughout, + cannot come apart that way. Ordinary reads leave it False and release, which is what + keeps a read from occupying a pool connection until its task ends. + """ + adapter = self._adapter + + tablenames = adapter.tables( + query, + attributes.get("join"), + attributes.get("left"), + attributes.get("orderby"), + attributes.get("groupby"), + ) + expanded_fields = adapter.expand_all(fields, tablenames) + colnames, sql = adapter._select_wcols(query, expanded_fields, **attributes) + + pool = await self._get_async_pool() + try: + async with pool.connection() as conn, conn.cursor() as cur: + await cur.execute(sql) + rows = await cur.fetchall() + finally: + if not _hold_connection: + await self._release_readonly_connection(pool) + + limitby = attributes.get("limitby") or (0,) + rows = adapter.rowslice(rows, limitby[0], None) + cacheable = attributes.get("cacheable", False) + return t.cast(pydal.objects.Rows, adapter.parse(rows, expanded_fields, colnames, cacheable=cacheable)) + + async def count_async( + self, + query: pydal.objects.Query, + distinct: t.Optional[bool] = None, + ) -> int: + """ + Async twin of `db(query).count(distinct)`. + + Mirrors `SQLAdapter.count()` (adapters/base.py): build via pydal's own + `_count()` (pure), execute via the async driver for this backend, read the first + column of the first (only) row. + """ + adapter = self._adapter + sql = adapter._count(query, distinct) + + pool = await self._get_async_pool() + try: + async with pool.connection() as conn, conn.cursor() as cur: + await cur.execute(sql) + row = await cur.fetchone() + finally: + await self._release_readonly_connection(pool) + + return t.cast(int, row[0]) + + async def update_async( + self, + table: pydal.objects.Table, + query: pydal.objects.Query, + fields: list[tuple[pydal.objects.Field, t.Any]], + ) -> t.Optional[int]: + """ + Async twin of the adapter-level step of `Set.update()` + (`adapter.update()`, adapters/base.py). + + `fields` is the already-normalized `[(Field, value), ...]` list (`row.op_values()`), + same shape as `insert_async`'s `fields` - the before_update/after_update hooks and + validation stay in `QueryBuilder.update_async()`, this is only the execute step. + """ + adapter = self._adapter + sql = adapter._update(table, query, fields) + + pool = await self._get_async_pool() + with self._mark_async_pending(): + async with pool.connection() as conn, conn.cursor() as cur: + try: + await cur.execute(sql) + except Exception as e: + if hasattr(table, "_on_update_error"): + return t.cast(t.Optional[int], table._on_update_error(table, query, fields, e)) # ty: ignore[call-non-callable] + raise + try: + return cur.rowcount + except Exception: # pragma: no cover + # defensive, mirroring `adapter.update()` (adapters/base.py): + # neither driver's `rowcount` actually raises, it is a plain property. + return None + + async def delete_async( + self, + table: pydal.objects.Table, + query: pydal.objects.Query, + ) -> t.Any: + """ + Async twin of `Set.delete()`'s adapter-level step (`adapter.delete()`). + + Dispatches per backend via `DELETE_STRATEGIES`: SQLite's isn't a plain + build/execute/parse call - it selects affected ids first and recurses for + ON DELETE CASCADE (adapters/sqlite.py) - Postgres's is. + """ + return await DELETE_STRATEGIES[self._adapter.dbengine](self, table, query) + + async def insert_async( + self, + table: pydal.objects.Table, + fields: list[tuple[pydal.objects.Field, t.Any]], + ) -> t.Any: + """ + Async twin of the adapter-level step of `table.insert(**fields)` + (`adapter.insert()`, adapters/base.py). + + `fields` is the already-normalized `[(Field, value), ...]` list (`row.op_values()`), + the same shape pydal's own `Table.insert()` passes to the adapter - the field-name-to- + value normalization, `_before_insert`/`_after_insert` hooks, and validation all stay in + `TypedTable.insert_async()` (tables.py), not here; this is only the execute step. + """ + adapter = self._adapter + query = adapter._insert(table, fields) + + # Capture `_last_insert` here, synchronously, right after the `_insert()` that set it: + # on Postgres it is a property over `THREAD_LOCAL._pydal_last_insert_` (pydal + # adapters/postgres.py), and coroutines share one thread, so that thread-local + # provides no isolation at all on this path. Reading it after the awaits below would + # read whichever concurrent insert_async() touched it last, not our own. + last_insert = getattr(adapter, "_last_insert", None) + + pool = await self._get_async_pool() + with self._mark_async_pending(): + async with pool.connection() as conn, conn.cursor() as cur: + try: + await cur.execute(query) + except Exception as e: + # mirrors `adapter.insert()` (adapters/base.py), same as `update_async`: + if hasattr(table, "_on_insert_error"): + return table._on_insert_error(table, fields, e) # ty: ignore[call-non-callable] + raise + + if hasattr(table, "_primarykey"): + pkdict = {k[0].name: k[1] for k in fields if k[0].name in table._primarykey} # ty: ignore[unsupported-operator] + if pkdict: + return pkdict + + row_id = await LASTROWID_STRATEGIES[adapter.dbengine](adapter, table, cur, last_insert) + + # a table with a single custom primarykey reports its id as a `{name: value}` dict + # instead of a bare int, matching `adapter.insert()` (adapters/base.py): + primarykey = getattr(table, "_primarykey", None) + if primarykey is not None and len(primarykey) == 1: # pragma: no cover + # unreachable on both supported backends: pydal makes `_primarykey` columns NOT + # NULL, so an insert omitting the pk fails in the database before the id it would + # have filled in here could ever be read back. Kept to match `adapter.insert()` + # (adapters/base.py) for backends that can generate one. + return {table._primarykey[0]: row_id} # ty: ignore[not-subscriptable] + + if not isinstance(row_id, int): # pragma: no cover + # a driver reporting no lastrowid at all; neither supported backend does. + return row_id + + reference = pydal.helpers.classes.Reference(row_id) # ty: ignore[possibly-missing-submodule] + reference._table = table + reference._record = None + return reference + + async def executesql_async( + self, + query: str | Template, + placeholders: t.Iterable[str] | dict[str, str] | None = None, + as_dict: bool = False, + fields: "Field | TypedField[t.Any] | Table | t.Iterable[Field | TypedField[t.Any]] | None" = None, + colnames: t.Iterable[str] | None = None, + as_ordered_dict: bool = False, + ) -> list[t.Any] | None: + """ + Async twin of `executesql(...)`. + + Mirrors pydal's own `DAL.executesql()` (base.py): execute via the async + driver for this backend (the only I/O), then the same as_dict/fields/colnames + branching pydal itself does, calling pydal's own `adapter.parse()` (pure) for the + fields/colnames case, unmodified. Only the plain-tuples path (no as_dict, no + fields/colnames) is covered by tests so far. + """ + if SYSTEM_SUPPORTS_TEMPLATES and isinstance(query, Template): # pragma: no cover + query = sql_escape_template(self, query) + + adapter = self._adapter + pool = await self._get_async_pool() + + # Unlike the other `_async` methods this one is handed arbitrary SQL, so whether it + # leaves uncommitted work is not knowable up front. Mark first and withdraw after: + # marking only once the statement has run would leave a window in which a sync write + # from another coroutine slips past the split guard, and reading the *statement text* + # to decide (as this used to do, and as the sync side still does) mistakes a leading + # comment, a CTE-wrapped write or DDL for a read - which then took the read-only + # release path below and silently rolled the write back. + left_uncommitted_work = False + left_work = UNCOMMITTED_WORK_STRATEGIES[adapter.dbengine] + + with self._mark_async_pending() as pending: + try: + async with pool.connection() as conn, conn.cursor() as cur: + if placeholders: + await cur.execute(query, placeholders) + else: + await cur.execute(query) + + # right after `execute()` and before any fetching, which is what both + # strategies read; fetching does not change either answer, but the early + # returns below would skip this. + left_uncommitted_work = left_work(conn, cur) + if not left_uncommitted_work: + pending.release() + + if as_dict or as_ordered_dict: + if not hasattr(cur, "description"): # pragma: no cover + # both supported drivers always expose it; guard kept for parity with + # pydal's own `executesql`. + raise RuntimeError("database does not support executesql_async(...,as_dict=True)") + + columns = cur.description + result_fields = list(colnames) if colnames else [col[0] for col in columns] + if len(result_fields) != len(set(result_fields)): + raise RuntimeError( + "Result set includes duplicate column names. " + "Specify unique column names using the 'colnames' argument", + ) + if columns: + for i in range(len(result_fields)): + if isinstance(result_fields[i], bytes): # pragma: no cover + # psycopg and aiosqlite both report column names as str; this is + # for drivers that hand back bytes, as pydal's `executesql` allows. + result_fields[i] = result_fields[i].decode("utf8") # ty: ignore[unresolved-attribute] + + data = await cur.fetchall() + _dict = collections.OrderedDict if as_ordered_dict else dict + return [_dict(zip(result_fields, row)) for row in data] + + try: + data = await cur.fetchall() + except Exception: + return None + except Exception: + # A statement that raised has no effect to preserve, and on Postgres it leaves + # the transaction aborted, so this task's mark goes with it - which lets the + # `finally` roll the connection back. Withdrawn rather than kept because the + # alternative is that one failed read blocks the sync side until the caller + # thinks to call `rollback_async()`. A task with *earlier* async writes is + # unaffected: `release()` leaves a mark it did not place. + pending.release() + raise + finally: + if not left_uncommitted_work: + await self._release_readonly_connection(pool) + + if fields or colnames: + if fields is None: + given_fields: list[t.Any] = [] + elif isinstance(fields, (pydal.objects.Expression, pydal.objects.Table, str, bytes)): + # pydal's `executesql` accepts one Field/Table instead of a list + # (base.py: `if not isinstance(fields, list): fields = [fields]`), and the sync + # `executesql()` above inherits that by delegating to it. Wrapping is not just + # for parity: `list()` on a single Field never terminates, because + # `Expression.__getitem__` (pydal objects.py) answers every integer index with a + # substring expression instead of raising IndexError, so iteration has no end. + # `str`/`bytes` are in here for the same reason in reverse: neither is valid + # input, but iterating one silently yields characters, so it would fail much + # later on `f.sqlsafe` with a character rather than with what was passed in. + given_fields = [fields] + else: + given_fields = list(fields) + extracted_fields: list[t.Any] = [] + for field in given_fields: + if isinstance(field, pydal.objects.Table): + extracted_fields.extend(list(field)) + else: + extracted_fields.append(field) + if not colnames: + resolved_colnames = [f.sqlsafe for f in extracted_fields] + else: + col_fields = [] + newcolnames = [] + for tf in colnames: + if "." in tf: + t_f = tf.split(".") + tf = ".".join(adapter.dialect.quote(f) for f in t_f) + else: + t_f = None + if not extracted_fields: + col_fields.append(t_f) + newcolnames.append(tf) + resolved_colnames = newcolnames + data = adapter.parse( + data, + fields=extracted_fields or [tf and self[tf[0]][tf[1]] for tf in col_fields], + colnames=resolved_colnames, + ) + + return t.cast(list[t.Any], data) + + async def commit_async(self) -> None: + """ + Commit the transaction on the async connection. + + Deliberately does not touch `commit()`/the sync connection: queries executed via + `select_async`/`insert_async`/etc. run on a separate connection, so committing one says + nothing about the other. On Postgres this ends the transaction on the connection + checked out for *this task* and returns it to the pool; on SQLite there is one + connection and it ends the only transaction there is. + + Goes to `AsyncPoolManager.pool` rather than `_get_async_pool()` on purpose: ending a + transaction must never be the thing that opens a connection, and it must stay callable + while the sync side has pending writes - the guard in `_get_async_pool()` would refuse + exactly when a caller is trying to settle up. + """ + if pool := self._async_pools.pool: + await pool.commit() + + # only this task's, matching what was actually committed: on Postgres `pool.commit()` + # ends the transaction on the connection checked out for *this* task and leaves every + # other task's alone. + self._async_pending_owners.discard(self._async_pending_owner()) + + async def rollback_async(self) -> None: + """ + Roll back the transaction on the async connection. See `commit_async`. + """ + if pool := self._async_pools.pool: + await pool.rollback() + + self._async_pending_owners.discard(self._async_pending_owner()) + + async def close_async(self) -> None: + """ + Close the async connection pool, if one was ever opened. + """ + await self._async_pools.close() + + # every connection those writes were sitting on is gone (rolled back on the way out), + # so nothing is pending anymore - and leaving stale owners behind would refuse sync + # statements on a database that no longer has an async side at all. + self._async_pending_owners.clear() + def sql_expression( self, sql_fragment: str | Template, @@ -630,7 +1196,7 @@ def memoize[T: t.Any]( Returns: Cached result or fresh computation """ - return memoize(self, func, *args, key=key, ttl=ttl, **kwargs) + return memoize(self, func, *args, key=key, ttl=ttl, **kwargs) # ty: ignore[invalid-argument-type] def as_typescript(self, *tables: str | type[TypedTable]) -> str: """ diff --git a/src/typedal/define.py b/src/typedal/define.py index f6cb107..8f4e658 100644 --- a/src/typedal/define.py +++ b/src/typedal/define.py @@ -37,7 +37,7 @@ from annotationlib import ForwardRef except ImportError: # pragma: no cover # python 3.13- - from typing import ForwardRef + from typing import ForwardRef # special case, keep `from typing` class IS_IN_ENUM(Validator): @@ -49,7 +49,7 @@ def __init__(self, etype: type[enum.Enum], error_message: str = "value not allow self.etype = etype self.error_message = error_message - def validate(self, value: t.Any, _record_id: int | None = None) -> t.Any: + def validate(self, value: t.Any, _record_id: int | None = None) -> t.Any: # ty: ignore[invalid-method-override] """Validate and normalize an enum-compatible value.""" if value not in self.etype: raise ValidationError(self.translator(self.error_message)) @@ -102,8 +102,8 @@ def define[T: t.Any](self, cls: t.Type[T], **kwargs: t.Unpack[DefineKwargs]) -> relationships |= { k: new_relationship for k in reference_field_keys - if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k])) - } + if k not in relationships and (new_relationship := to_relationship(cls, k, annotations[k])) # ty: ignore[invalid-argument-type] + } # ty: ignore[unsupported-operator] cache_dependency = self.db._config.caching and kwargs.pop("cache_dependency", True) table: Table = self.db.define_table(tablename, *fields.values(), **kwargs) @@ -131,7 +131,7 @@ def define[T: t.Any](self, cls: t.Type[T], **kwargs: t.Unpack[DefineKwargs]) -> table._after_insert.append(lambda _row, _id: remove_cache_for_table(tablename)) table._before_update.append(lambda s, _: _remove_cache(s, tablename)) - table._before_delete.append(lambda s: _remove_cache(s, tablename)) + table._before_delete.append(lambda s: _remove_cache(s, tablename)) # ty: ignore[invalid-argument-type] return cls diff --git a/src/typedal/fields.py b/src/typedal/fields.py index 672368c..f2fbe70 100644 --- a/src/typedal/fields.py +++ b/src/typedal/fields.py @@ -195,7 +195,7 @@ def bind(self, field: pydal.objects.Field, table: pydal.objects.Table) -> None: Bind the right db/table/field info to this class, so queries can be made using `Class.field == ...`. """ self._table = table - self._field = field + self._field = t.cast(Field, field) def unbind(self) -> None: """Remove references to the pydal objects created during ``bind``.""" @@ -219,37 +219,37 @@ def __getattr__(self, key: str) -> t.Any: # try on actual field: return getattr(self._field, key) - def __eq__(self, other: t.Any) -> Query: + def __eq__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing == on a Field will result in a Query. """ return t.cast(Query, self._field == other) - def __ne__(self, other: t.Any) -> Query: + def __ne__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing != on a Field will result in a Query. """ return t.cast(Query, self._field != other) - def __gt__(self, other: t.Any) -> Query: + def __gt__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing > on a Field will result in a Query. """ return t.cast(Query, self._field > other) - def __lt__(self, other: t.Any) -> Query: + def __lt__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing < on a Field will result in a Query. """ return t.cast(Query, self._field < other) - def __ge__(self, other: t.Any) -> Query: + def __ge__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing >= on a Field will result in a Query. """ return t.cast(Query, self._field >= other) - def __le__(self, other: t.Any) -> Query: + def __le__(self, other: t.Any) -> Query: # ty: ignore[invalid-method-override] """ Performing <= on a Field will result in a Query. """ @@ -604,7 +604,7 @@ def safe_encode_native_point(value: tuple[str, str] | tuple[float, float] | str) return "" value_tup = tuple(float(x.strip()) for x in value.split(",")) else: - value_tup = value # type: ignore + value_tup = value # Validate and format if len(value_tup) != 2: diff --git a/src/typedal/for_py4web.py b/src/typedal/for_py4web.py index 446b072..885f3d9 100644 --- a/src/typedal/for_py4web.py +++ b/src/typedal/for_py4web.py @@ -2,7 +2,7 @@ ONLY USE IN COMBINATION WITH PY4WEB! """ -import typing +import typing as t import threadsafevariable from py4web.core import ICECUBE @@ -21,9 +21,9 @@ class Fixture(_Fixture): class PY4WEB_DAL_SINGLETON(MetaDAL): - _instances: typing.ClassVar[typing.MutableMapping[str, TypeDAL]] = {} + _instances: t.ClassVar[t.MutableMapping[str, TypeDAL]] = {} - def __call__(cls, uri: typing.Optional[str] = None, *args: typing.Any, **kwargs: typing.Any) -> TypeDAL: + def __call__(cls, uri: str | None = None, *args: t.Any, **kwargs: t.Any) -> TypeDAL: db_uid = kwargs.get("db_uid", hashlib_md5(repr(uri or (args, kwargs))).hexdigest()) if db_uid not in cls._instances: cls._instances[db_uid] = super().__call__(uri, *args, **kwargs) @@ -39,20 +39,20 @@ class DAL(TypeDAL, Fixture, metaclass=PY4WEB_DAL_SINGLETON): # pragma: no cover Fixture similar to the py4web pydal fixture, but for typedal. """ - def on_request(self, _: AnyDict) -> None: + def on_request(self, _: AnyDict) -> None: # ty: ignore[invalid-method-override] """ Make sure there is a database connection when a request comes in. """ self.get_connection_from_pool_or_new() threadsafevariable.ThreadSafeVariable.restore(ICECUBE) - def on_error(self, _: AnyDict) -> None: + def on_error(self, _: AnyDict) -> None: # ty: ignore[invalid-method-override] """ Rollback db on error. """ self.recycle_connection_in_pool_or_close("rollback") - def on_success(self, _: AnyDict) -> None: + def on_success(self, _: AnyDict) -> None: # ty: ignore[invalid-method-override] """ Commit db on success. """ diff --git a/src/typedal/for_web2py.py b/src/typedal/for_web2py.py index 0cd9fd8..2c7a320 100644 --- a/src/typedal/for_web2py.py +++ b/src/typedal/for_web2py.py @@ -3,11 +3,13 @@ """ import datetime as dt +import typing as t from pydal.validators import IS_NOT_IN_DB from . import TypeDAL, TypedField, TypedTable from .fields import TextField +from .types import Validator from .web2py_py4web_shared import AuthUser DAL = TypeDAL # export as DAL for compatibility with py4web @@ -28,7 +30,8 @@ def __on_define__(cls, db: TypeDAL) -> None: """ super().__on_define__(db) - cls.role.requires = IS_NOT_IN_DB(db, "w2p_auth_group.role") + requires = [IS_NOT_IN_DB(db, "w2p_auth_group.role")] + cls.role.requires = t.cast(list[Validator], requires) class AuthMembership(TypedTable): diff --git a/src/typedal/helpers.py b/src/typedal/helpers.py index 40211a1..4f2f87d 100644 --- a/src/typedal/helpers.py +++ b/src/typedal/helpers.py @@ -191,7 +191,7 @@ def filter_out[K, V, T](mut_dict: dict[K, V], _type: type[T]) -> dict[K, T]: Modifies mut_dict and returns everything of type _type. """ - return {k: mut_dict.pop(k) for k, v in list(mut_dict.items()) if looks_like(v, _type)} + return t.cast(dict[K, T], {k: mut_dict.pop(k) for k, v in list(mut_dict.items()) if looks_like(v, _type)}) def unwrap_type(_type: type) -> type: @@ -311,7 +311,7 @@ def get_table(table: "TypedTable | Table") -> "Table": """ Get the underlying pydal table for a typedal table. """ - return t.cast("Table", table._table) + return t.cast("Table", table._table) # ty: ignore[unresolved-attribute] def get_field(field: "TypedField[t.Any] | Field") -> "Field": diff --git a/src/typedal/mixins.py b/src/typedal/mixins.py index dfbf278..01006c6 100644 --- a/src/typedal/mixins.py +++ b/src/typedal/mixins.py @@ -105,7 +105,7 @@ def __init__( """ super().__init__(db, field, error_message) - def validate[T](self, original: T, record_id: t.Optional[int] = None) -> T: + def validate[T](self, original: T, record_id: t.Optional[int] = None) -> T: # ty: ignore[invalid-method-override] """ Performs checks to see if the slug already exists for a different row. """ @@ -261,7 +261,7 @@ def model_dump(self, mode: str = "python", **kwargs: t.Any) -> dict[str, t.Any]: try: from pydantic import BaseModel except ImportError: - BaseModel = BaseModeProtocol # type: ignore + BaseModel = BaseModeProtocol def dump_pydantic[T](values: T, _shallow_nested: bool = False) -> T: @@ -367,7 +367,7 @@ def _pydantic_fields( @staticmethod def _make_instance_converter(model_cls: type, fields: dict[str, t.Any]) -> t.Callable[[t.Any], t.Any]: _PRIMITIVES = (str, float, bool, bytes) - relationship_names = set(model_cls.get_relationships()) if hasattr(model_cls, "get_relationships") else set() + relationship_names = set(model_cls.get_relationships()) if hasattr(model_cls, "get_relationships") else set() # ty: ignore[call-non-callable] def convert(value: t.Any) -> t.Any: if isinstance(value, dict): @@ -534,7 +534,7 @@ def __get_pydantic_json_schema__( handler: t.Any, ) -> dict[str, t.Any]: """Build the JSON schema by delegating to pydantic's handler.""" - return handler(schema) # type: ignore + return handler(schema) def model_dump(self, mode: str = "python", *, _shallow: bool = False) -> dict[str, t.Any]: """Serialize this model to a dict, with optional shallow nested output.""" diff --git a/src/typedal/query_builder.py b/src/typedal/query_builder.py index 4dd07ad..5e90086 100644 --- a/src/typedal/query_builder.py +++ b/src/typedal/query_builder.py @@ -61,11 +61,11 @@ class QueryBuilder[T_MetaInstance: _TypedTable]: def __init__( self, model: t.Type[T_MetaInstance], - add_query: t.Optional[Query] = None, - select_args: t.Optional[list[t.Any]] = None, - select_kwargs: t.Optional[SelectKwargs] = None, - relationships: dict[str, Relationship[t.Any]] = None, - metadata: Metadata = None, + add_query: Query | None = None, + select_args: list[t.Any] | None = None, + select_kwargs: SelectKwargs | None = None, + relationships: dict[str, Relationship[t.Any]] | None = None, + metadata: Metadata | None = None, permissions: Permissions | None = None, ): """ @@ -76,7 +76,7 @@ def __init__( """ self.model = model table = self._ensure_table_defined() - default_query: Query = t.cast(Query, table.id > 0) + default_query: Query = t.cast(Query, table.id > 0) # ty: ignore[unresolved-attribute] self.query = add_query or default_query self.select_args = select_args or [] self.select_kwargs = select_kwargs or {} @@ -117,7 +117,7 @@ def __bool__(self) -> bool: Querybuilder is truthy if it has t.Any conditions. """ table = self._ensure_table_defined() - default_query: Query = t.cast(Query, table.id > 0) + default_query: Query = t.cast(Query, table.id > 0) # ty: ignore[unresolved-attribute] return any( [ self.query != default_query, @@ -130,12 +130,12 @@ def __bool__(self) -> bool: def _extend( self, - add_query: t.Optional[Query] = None, - overwrite_query: t.Optional[Query] = None, - select_args: t.Optional[list[t.Any]] = None, - select_kwargs: t.Optional[SelectKwargs] = None, - relationships: dict[str, Relationship[t.Any]] = None, - metadata: Metadata = None, + add_query: Query | None = None, + overwrite_query: Query | None = None, + select_args: list[t.Any] | None = None, + select_kwargs: SelectKwargs | None = None, + relationships: dict[str, Relationship[t.Any]] | None = None, + metadata: Metadata | None = None, permissions: Permissions | None = None, ) -> "QueryBuilder[T_MetaInstance]": return QueryBuilder( @@ -144,7 +144,7 @@ def _extend( (self.select_args + select_args) if select_args else self.select_args, (self.select_kwargs | select_kwargs) if select_kwargs else self.select_kwargs, (self.relationships | relationships) if relationships else self.relationships, - (self.metadata | (metadata or {})) if metadata else self.metadata, + (self.metadata | (metadata or {})) if metadata else self.metadata, # ty: ignore[invalid-argument-type] permissions=merge_permissions(self._permissions, permissions), ) @@ -162,7 +162,7 @@ def _normalize_select_option( return value if isinstance(value, (list, tuple, set)): - return list(self._normalize_select_option(val) for val in value) + return t.cast(list[str], [self._normalize_select_option(val) for val in value]) if rname := getattr(value, "_rname", None): return str(rname) @@ -275,7 +275,7 @@ def where( elif isinstance(query_part, (pydal.objects.Query, Expression, pydal.objects.Expression)): subquery |= t.cast(Query, query_part) elif callable(query_part): - if result := query_part(self.model): + if result := query_part(self.model): # ty: ignore[call-top-callable] subquery |= result elif isinstance(query_part, dict): subsubquery = DummyQuery() @@ -399,10 +399,10 @@ def join( raise ValueError("join(field, on=...) can only be used with exactly one field!") if isinstance(on, pydal.objects.Expression): - on = [on] + on = t.cast(list[Expression], [on]) if isinstance(on, list): - on = as_lambda(on) + on = t.cast(OnQuery, as_lambda(on)) field = fields[0] if isinstance(field, Relationship) and field.name: @@ -505,6 +505,41 @@ def _delete(self) -> str: db = self._get_db() return str(db(self.query)._delete()) + async def delete_async(self) -> list[int]: + """ + Async twin of `delete()`. + + `delete()` delegates the before_delete/after_delete hook dance to pydal's own + `Set.delete()` (objects.py); since pydal has no async version of that to + delegate to, it's replicated here, same reasoning as `insert_async` - only the + adapter-level execute step (`db.delete_async(...)`) is async. + """ + require_permission(self._permissions, "delete") + db = self._get_db() + + # `_hold_connection`: the delete below has to land in the same transaction as this + # snapshot, or the ids returned describe rows it never touched - see `select_async`. + removed_rows = await db.select_async(self.query, "id", _hold_connection=True) + removed_ids = [row.id for row in removed_rows] + + pydal_set = db(self.query) + table = db._adapter.get_table(self.query) + + if any(f(pydal_set) for f in table._before_delete): + # the delete the snapshot above was holding its connection for is not happening + await db._release_held_connection() + return [] + + result = await db.delete_async(table, self.query) + + if result: + # success! + for f in table._after_delete: + f(pydal_set) + return removed_ids + + return [] + def update(self, **fields: t.Any) -> list[int]: """ Based on the current query, update `fields` and return a list of updated IDs. @@ -523,6 +558,46 @@ def _update(self, **fields: t.Any) -> str: db = self._get_db() return str(db(self.query)._update(**fields)) + async def update_async(self, **fields: t.Any) -> list[int]: + """ + Async twin of `update(**fields)`. + + `update()` delegates the before_update/after_update hook dance to pydal's own + `Set.update()` (objects.py: `_build_update_row`/`_apply_update`); since pydal has no + async version of that to delegate to, it's replicated here, same reasoning as + `insert_async`/`delete_async` - only the adapter-level execute step + (`db.update_async(...)`) is async. + """ + require_permission(self._permissions, "update") + db = self._get_db() + + # `_hold_connection`: same reason as in `delete_async` - the update below has to share + # this snapshot's transaction, see `select_async`. + updated_rows = await db.select_async(self.query, "id", _hold_connection=True) + updated_ids = [row.id for row in updated_rows] + + pydal_set = db(self.query) + table = db._adapter.get_table(self.query) + row = table._fields_and_values_for_update(fields) + if not row._values: + await db._release_held_connection() + raise ValueError("No fields to update") + + if any(f(pydal_set, row) for f in table._before_update): + # the update the snapshot above was holding its connection for is not happening + await db._release_held_connection() + return [] + + result = await db.update_async(table, self.query, row.op_values()) + + if result: + # success! + for f in table._after_update: + f(pydal_set, row) + return updated_ids + + return [] + def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[t.Any], SelectKwargs]: select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL] select_kwargs = self.select_kwargs.copy() @@ -588,30 +663,106 @@ def _collect_cached( return load_from_cache(key, self._get_db()) - def execute(self, add_id: bool = False) -> Rows: + @staticmethod + def _run_hooks(hooks: t.Iterable[t.Callable[..., t.Any]], *args: t.Any) -> None: """ - Raw version of .collect which only executes the SQL, without performing t.Any magic afterwards. + Run a list of before/after hooks in order. Return values are ignored (matches existing + `_before_collect`/`_after_collect`/`_before_execute`/`_after_execute` semantics). + Shared by `execute()`/`execute_async()`/`collect()`/`collect_async()`. + """ + for hook in hooks: + hook(*args) + + def _execute_prepare(self, metadata: Metadata, add_id: bool) -> tuple[TypeDAL, Query, list[t.Any], SelectKwargs]: + """ + Shared setup for `execute()`/`execute_async()`: permission check, query building. """ require_permission(self._permissions, "read") db = self._get_db() - metadata: Metadata = self.metadata.copy() - query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id) + return db, query, select_args, select_kwargs - for fn_before in db._before_execute: - fn_before(self) + def execute(self, add_id: bool = False) -> Rows: + """ + Raw version of .collect which only executes the SQL, without performing t.Any magic afterwards. + """ + db, query, select_args, select_kwargs = self._execute_prepare(self.metadata.copy(), add_id) + self._run_hooks(db._before_execute, self) rows: Rows = db(query).select(*select_args, **select_kwargs) + self._run_hooks(db._after_execute, self, rows) + + return rows + + async def execute_async(self, add_id: bool = False) -> Rows: + """ + Async twin of `execute()`. + """ + db, query, select_args, select_kwargs = self._execute_prepare(self.metadata.copy(), add_id) - for fn_after in db._after_execute: - fn_after(self, rows) + self._run_hooks(db._before_execute, self) + rows: Rows = await db.select_async(query, *select_args, **select_kwargs) + self._run_hooks(db._after_execute, self, rows) return rows + def _collect_prepare( + self, + metadata: Metadata, + add_id: bool, + into: t.Type[t.Any], + ) -> "TypedRows[T_MetaInstance] | tuple[TypeDAL, Query, list[t.Any], SelectKwargs]": + """ + Shared setup for `collect()`/`collect_async()`, up to (not including) the actual select. + + Returns a `TypedRows` directly if a cache hit short-circuits everything else, + otherwise the `(db, query, select_args, select_kwargs)` needed to perform the fetch. + """ + require_permission(self._permissions, "read") + db = self._get_db() + self._run_hooks(db._before_collect, self) + + if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata, into)): + return result + + query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id) + metadata["sql"] = db(query)._select(*select_args, **select_kwargs) + + return db, query, select_args, select_kwargs + + @staticmethod + def _record_fetch_metadata( + metadata: Metadata, + query: Query, + select_args: list[t.Any], + select_kwargs: SelectKwargs, + duration: float, + ) -> None: + """ + Shared metadata bookkeeping after a fetch, for `collect()`/`collect_async()`. + """ + metadata["final_query"] = str(query) + metadata["final_args"] = [str(_) for _ in select_args] + metadata["final_kwargs"] = select_kwargs + metadata["select_duration"] = duration + + def _finalize_collect( + self, + typed_rows: TypedRows[T_MetaInstance], + rows: Rows, + db: TypeDAL, + ) -> TypedRows[T_MetaInstance]: + """ + Shared tail of `collect()`/`collect_async()`: after_collect hooks + cache save. + """ + self._run_hooks(db._after_collect, self, typed_rows, rows) + # only saves if requested in metadata: + return save_to_cache(typed_rows, rows) # ty: ignore[invalid-argument-type] + def collect( self, verbose: bool = False, - _to: t.Type["TypedRows[t.Any]"] = None, + _to: t.Type["TypedRows[t.Any]"] | None = None, add_id: bool = True, _into: t.Type[_TypedTable] | None = None, _init: t.Callable[[_TypedTable, Row], None] | None = None, @@ -619,7 +770,6 @@ def collect( """ Execute the built query and turn it into model instances, while handling relationships. """ - require_permission(self._permissions, "read") if _to is None: _to = TypedRows into = _into or self.model @@ -627,33 +777,20 @@ def collect( if not isinstance(self.model, TableMeta): # tried to use querybuilder with a non-typedal table, # fallback to execute: - return self.execute(add_id=add_id) - - db = self._get_db() - - for fn_before in db._before_collect: - fn_before(self) + return t.cast(TypedRows[T_MetaInstance], self.execute(add_id=add_id)) metadata: Metadata = self.metadata.copy() - - if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata, into)): - return result - - query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id) - - metadata["sql"] = db(query)._select(*select_args, **select_kwargs) + prepared = self._collect_prepare(metadata, add_id, into) + if not isinstance(prepared, tuple): + return prepared + db, query, select_args, select_kwargs = prepared if verbose: # pragma: no cover print(metadata["sql"]) start_time = time.perf_counter() rows: Rows = db(query).select(*select_args, **select_kwargs) - duration = time.perf_counter() - start_time - - metadata["final_query"] = str(query) - metadata["final_args"] = [str(_) for _ in select_args] - metadata["final_kwargs"] = select_kwargs - metadata["select_duration"] = duration + self._record_fetch_metadata(metadata, query, select_args, select_kwargs, time.perf_counter() - start_time) if verbose: # pragma: no cover print(rows) @@ -667,11 +804,58 @@ def collect( # if that's not the case, return default behavior again typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to, _into=into, _init=_init) - for fn_after in db._after_collect: - fn_after(self, typed_rows, rows) + return self._finalize_collect(typed_rows, rows, db) - # only saves if requested in metadata: - return save_to_cache(typed_rows, rows) + async def collect_async( + self, + verbose: bool = False, + _to: t.Type["TypedRows[t.Any]"] | None = None, + add_id: bool = True, + _into: t.Type[_TypedTable] | None = None, + _init: t.Callable[[_TypedTable, Row], None] | None = None, + ) -> TypedRows[T_MetaInstance]: + """ + Async twin of `collect()`: same shape, only the execute step in the middle is async. + + Relationships/joins included: nothing on that path executes a second query. The joins + are built into the one query by `_before_query()` (already shared via `_collect_prepare`), + `_apply_limitby_optimization()` only *generates* SQL (`db(query)._select(...)`, inlined + as a subquery - no execution), and `_collect_with_relationships()` maps rows already + fetched here into instances. So the same tail works for both paths. + """ + if _to is None: + _to = TypedRows + into = _into or self.model + + if not isinstance(self.model, TableMeta): + # tried to use querybuilder with a non-typedal table, + # fallback to execute: + return t.cast(TypedRows[T_MetaInstance], await self.execute_async(add_id=add_id)) + + metadata: Metadata = self.metadata.copy() + prepared = self._collect_prepare(metadata, add_id, into) + if not isinstance(prepared, tuple): + return prepared + db, query, select_args, select_kwargs = prepared + + if verbose: # pragma: no cover + print(metadata["sql"]) + + start_time = time.perf_counter() + rows: Rows = await db.select_async(query, *select_args, **select_kwargs) + self._record_fetch_metadata(metadata, query, select_args, select_kwargs, time.perf_counter() - start_time) + + if verbose: # pragma: no cover + print(rows) + + if not self.relationships: + # easy + typed_rows = _to.from_rows(rows, self.model, metadata=metadata, into=into, init=_init) + else: + # harder: try to match rows to the belonging objects + typed_rows = self._collect_with_relationships(rows, metadata=metadata, _to=_to, _into=into, _init=_init) + + return self._finalize_collect(typed_rows, rows, db) def collect_into[T_Into: _TypedTable]( self, @@ -691,6 +875,24 @@ def collect_into[T_Into: _TypedTable]( rows = query.collect(verbose=verbose, add_id=add_id, _into=into, _init=_init) return t.cast(TypedRows[T_Into], rows) + async def collect_into_async[T_Into: _TypedTable]( + self, + into: t.Type[T_Into], + verbose: bool = False, + add_id: bool = True, + init: t.Callable[[T_Into, Row], None] | None = None, + ) -> TypedRows[T_Into]: + """ + Async twin of `collect_into()`. Thin wrapper: builds on `collect_async()`. + """ + self._validate_collect_into_model(into) + query = self + if not self.select_args: + query = self.select(*self._collect_into_default_fields(into)) + _init = t.cast(t.Callable[[_TypedTable, Row], None] | None, init) + rows = await query.collect_async(verbose=verbose, add_id=add_id, _into=into, _init=_init) + return t.cast(TypedRows[T_Into], rows) + def _validate_collect_into_model(self, into: t.Type[t.Any]) -> None: if not isinstance(into, TableMeta): raise TypeError("collect_into expects a TypedTable class") @@ -734,6 +936,13 @@ def column[T: t.Any](self, field: TypedField[T] | T, **options: t.Unpack[SelectK """ return self.select(field, **options).execute().column(field) + async def column_async[T: t.Any](self, field: TypedField[T] | T, **options: t.Unpack[SelectKwargs]) -> list[T]: + """ + Async twin of `column()`. Thin wrapper: `.select(field).execute_async()` then `.column(field)`. + """ + rows = await self.select(field, **options).execute_async() + return t.cast(list[T], rows.column(field)) + def _handle_relationships_pre_select( self, query: Query, @@ -803,10 +1012,10 @@ def _build_inner_joins_recursive( if relation.condition and relation.join == "inner": other = relation.get_table(db) other = other.with_alias(f"{key}_{hash(relation)}") - condition = relation.condition(parent_table, other) + condition = relation.condition(parent_table, other) # ty: ignore[invalid-argument-type] if callable(relation.condition_and): - condition &= relation.condition_and(parent_table, other) + condition &= relation.condition_and(parent_table, other) # ty: ignore[invalid-argument-type] joins.append(other.on(condition)) @@ -837,7 +1046,7 @@ def _selectable_orderby_fields(self, orderby: OrderBy | t.Iterable[OrderBy] | No return [expression_without_direction if direction.upper() in {"ASC", "DESC"} else orderby] if isinstance(orderby, pydal.objects.Field): - return [orderby] + return t.cast(list[OrderBy], [orderby]) fields = [] first = getattr(orderby, "first", None) @@ -931,19 +1140,19 @@ def _process_relationship_for_left_join( # Build join condition if relation.on: # Custom .on condition - always left join - on = relation.on(parent_table, other) + on = relation.on(parent_table, other) # ty: ignore[invalid-argument-type] if not isinstance(on, list): on = [on] on = [_ for _ in on if isinstance(_, pydal.objects.Expression)] - left_joins.extend(on) + left_joins.extend(on) # ty: ignore[invalid-argument-type] elif method == "left": # Generate left join condition other = other.with_alias(f"{key}_{hash(relation)}") - condition = t.cast(Query, relation.condition(parent_table, other)) + condition = t.cast(Query, relation.condition(parent_table, other)) # ty: ignore[call-non-callable, invalid-argument-type] if callable(relation.condition_and): - condition &= relation.condition_and(parent_table, other) + condition &= relation.condition_and(parent_table, other) # ty: ignore[invalid-argument-type] left_joins.append(other.on(condition)) else: @@ -1181,6 +1390,12 @@ def collect_or_fail(self, exception: t.Optional[Exception] = None) -> TypedRows[ """ return self.collect() or throw(exception or ValueError("Nothing found!")) + async def collect_or_fail_async(self, exception: t.Optional[Exception] = None) -> TypedRows[T_MetaInstance]: + """ + Async twin of `collect_or_fail()`. Thin wrapper: builds on `collect_async()`. + """ + return await self.collect_async() or throw(exception or ValueError("Nothing found!")) + def __iter__(self) -> t.Generator[T_MetaInstance, None, None]: """ You can start iterating a Query Builder object before calling collect, for ease of use. @@ -1211,20 +1426,33 @@ def __count( other = other.with_alias(f"{key}_{hash(relation)}") if relation.condition is not None: - query &= relation.condition(model, other) + query &= relation.condition(model, other) # ty: ignore[invalid-argument-type] return query - def count(self, distinct: t.Optional[bool] = None) -> int: + def _count_prepare(self, distinct: t.Optional[bool] = None) -> tuple[TypeDAL, Query]: """ - Return the amount of rows matching the current query. + Shared setup for `count()`/`count_async()`. """ require_permission(self._permissions, "read") db = self._get_db() query = self.__count(db, distinct=distinct) + return db, query + def count(self, distinct: t.Optional[bool] = None) -> int: + """ + Return the amount of rows matching the current query. + """ + db, query = self._count_prepare(distinct) return db(query).count(distinct) + async def count_async(self, distinct: t.Optional[bool] = None) -> int: + """ + Async twin of `count()`. + """ + db, query = self._count_prepare(distinct) + return await db.count_async(query, distinct) + def _count(self, distinct: t.Optional[bool] = None) -> str: """ Return the SQL for .count(). @@ -1246,21 +1474,51 @@ def exists(self) -> bool: require_permission(self._permissions, "read") return bool(self.count()) + async def exists_async(self) -> bool: + """ + Async twin of `exists()`. Thin wrapper: builds on `count_async()`. + """ + require_permission(self._permissions, "read") + return bool(await self.count_async()) + + def __pagination_count_query(self) -> tuple[TypeDAL, Query]: + """ + Shared setup for `__pagination_count()`/`__pagination_count_async()`, for the + relationship case only (without relationships both just defer to `count()`). + """ + db = self._get_db() + query = self.__count(db, distinct=self.model.id, include_left_for_distinct=False) # ty: ignore[invalid-argument-type] + return db, query + def __pagination_count(self) -> int: if not self.relationships: return self.count() - db = self._get_db() - query = self.__count(db, distinct=self.model.id, include_left_for_distinct=False) + db, query = self.__pagination_count_query() return db(query).count(self.model.id) - def __paginate( + async def __pagination_count_async(self) -> int: + """ + Async twin of `__pagination_count()`: the row-count step `paginate_async()` needs + before it can know `max_page`. + """ + if not self.relationships: + return await self.count_async() + + db, query = self.__pagination_count_query() + return await db.count_async(query, self.model.id) + + def __paginate_builder( self, + available: int, limit: int, page: int = 1, ) -> "QueryBuilder[T_MetaInstance]": - available = self.__pagination_count() - + """ + Shared tail of `__paginate()`/`paginate_async()`: turn an already-determined row count + into a limitby-extended builder. Split out because the count step differs (sync vs async), + the metadata bookkeeping around it does not. + """ _from = limit * (page - 1) _to = (limit * page) if limit else available @@ -1276,6 +1534,20 @@ def __paginate( return self._extend(select_kwargs={"limitby": (_from, _to)}, metadata=metadata) + def __paginate( + self, + limit: int, + page: int = 1, + ) -> "QueryBuilder[T_MetaInstance]": + return self.__paginate_builder(self.__pagination_count(), limit, page) + + async def __paginate_async( + self, + limit: int, + page: int = 1, + ) -> "QueryBuilder[T_MetaInstance]": + return self.__paginate_builder(await self.__pagination_count_async(), limit, page) + def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]": """ Paginate transforms the more readable `page` and `limit` to pydals internal limit and offset. @@ -1291,6 +1563,24 @@ def paginate(self, limit: int, page: int = 1, verbose: bool = False) -> "Paginat rows._query_builder = builder return rows + async def paginate_async(self, limit: int, page: int = 1, verbose: bool = False) -> "PaginatedRows[T_MetaInstance]": + """ + Async twin of `paginate()`. Thin wrapper: builds on `collect_async()`. + + Note: `__pagination_count()` (the row-count step done before paginating) also hits the + DB and needs its own async path internally - not exposed as a separate public method. + """ + require_permission(self._permissions, "read") + builder = await self.__paginate_async(limit, page) + + rows = t.cast( + PaginatedRows[T_MetaInstance], + await builder.collect_async(verbose=verbose, _to=PaginatedRows), + ) + + rows._query_builder = builder + return rows + def _paginate( self, limit: int, @@ -1321,6 +1611,22 @@ def chunk(self, chunk_size: int) -> t.Generator[TypedRows[T_MetaInstance], t.Any yield rows page += 1 + async def chunk_async(self, chunk_size: int) -> t.AsyncGenerator[TypedRows[T_MetaInstance], None]: + """ + Async twin of `chunk()`. An async generator (`async for`), built on `collect_async()`. + """ + require_permission(self._permissions, "read") + page = 1 + + while True: + builder = await self.__paginate_async(chunk_size, page) + rows = await builder.collect_async() + if not rows: + return + + yield rows + page += 1 + def first(self, verbose: bool = False) -> T_MetaInstance | None: """ Get the first row matching the currently built query. @@ -1336,7 +1642,23 @@ def first(self, verbose: bool = False) -> T_MetaInstance | None: # old-style pydal table: keep pydal semantics and return raw Row return row - return self.model.from_row(row) + return self.model.from_row(row) # ty: ignore[invalid-argument-type] + + async def first_async(self, verbose: bool = False) -> T_MetaInstance | None: + """ + Async twin of `first()`. Thin wrapper: builds on `paginate_async()`. + """ + require_permission(self._permissions, "read") + paginated = await self.paginate_async(page=1, limit=1, verbose=verbose) + row = paginated.first() + if not row: + return None + + if not isinstance(self.model, TableMeta): + # old-style pydal table: keep pydal semantics and return raw Row + return row + + return self.model.from_row(row) # ty: ignore[invalid-argument-type] def _first(self) -> str: return self._paginate(page=1, limit=1) @@ -1350,6 +1672,15 @@ def first_or_fail(self, exception: t.Optional[BaseException] = None, verbose: bo require_permission(self._permissions, "read") return self.first(verbose=verbose) or throw(exception or ValueError("Nothing found!")) + async def first_or_fail_async( + self, exception: t.Optional[BaseException] = None, verbose: bool = False + ) -> T_MetaInstance: + """ + Async twin of `first_or_fail()`. Thin wrapper: builds on `first_async()`. + """ + require_permission(self._permissions, "read") + return await self.first_async(verbose=verbose) or throw(exception or ValueError("Nothing found!")) + # note: these imports exist at the bottom of this file to prevent circular import issues: diff --git a/src/typedal/relationships.py b/src/typedal/relationships.py index d015b35..309818a 100644 --- a/src/typedal/relationships.py +++ b/src/typedal/relationships.py @@ -5,13 +5,12 @@ import inspect import typing as t import warnings -from typing import ForwardRef import pydal.objects from .config import LazyPolicy from .constants import JOIN_OPTIONS -from .core import TypeDAL, evaluate_forward_reference +from .core import ForwardRef, TypeDAL, evaluate_forward_reference from .fields import TypedField from .helpers import extract_type_optional, looks_like, unwrap_type from .types import Condition, OnQuery, T_Field @@ -48,7 +47,7 @@ def __init__( join: JOIN_OPTIONS = None, on: OnQuery = None, condition_and: Condition = None, - nested: dict[str, t.Self] = None, + nested: dict[str, t.Self] | None = None, lazy: LazyPolicy | None = None, explicit: bool = False, ): @@ -150,7 +149,7 @@ def get_table(self, db: "TypeDAL") -> t.Type["TypedTable"]: # boo, fall back to untyped table but pretend it is typed: return t.cast(t.Type["TypedTable"], db[table]) # eh close enough! - return table + return t.cast(t.Type["TypedTable"], table) def get_db(self) -> TypeDAL | None: """ @@ -552,7 +551,7 @@ def resolve_relationship_type( if any(a is None for a in resolved_args): return None if origin is list: - return list[resolved_args[0]] # type: ignore[valid-type] + return list[resolved_args[0]] # type: ignore[valid-type] # ty: ignore[invalid-type-form] # Other generics: return as-is (already resolvable) return relationship_type diff --git a/src/typedal/rows.py b/src/typedal/rows.py index 0ccb142..830797a 100644 --- a/src/typedal/rows.py +++ b/src/typedal/rows.py @@ -49,9 +49,9 @@ def __init__( self, rows: Rows, model: t.Type[T_MetaInstance], - records: dict[int, T_MetaInstance] = None, - metadata: Metadata = None, - raw: dict[int, list[Row]] = None, + records: dict[int, T_MetaInstance] | None = None, + metadata: Metadata | None = None, + raw: dict[int, list[Row]] | None = None, ) -> None: """ Should not be called manually! @@ -127,7 +127,7 @@ def last(self) -> T_MetaInstance | None: def find( self, f: t.Callable[[T_MetaInstance], Query], - limitby: tuple[int, int] = None, + limitby: tuple[int, int] | None = None, ) -> "TypedRows[T_MetaInstance]": """ Returns a new Rows object, a subset of the original object, filtered by the function `f`. @@ -198,7 +198,7 @@ def __repr__(self) -> str: return mktable(data, headers) - def group_by_value[T: t.Any, T_MetaInstance: _TypedTable]( + def group_by_value[T: t.Any, T_MetaInstance: _TypedTable]( # ty: ignore[shadowed-type-variable] self, *fields: "str | Field | TypedField[T]", one_result: bool = False, @@ -246,8 +246,8 @@ def as_dict( return {k: v.as_dict() for k, v in self.records.items()} - def as_json( - self, default: t.Callable[[t.Any], t.Any] = None, indent: t.Optional[int] = None, **kwargs: t.Any + def as_json( # ty: ignore[invalid-method-override] + self, default: t.Callable[[t.Any], t.Any] | None = None, indent: int | None = None, **kwargs: t.Any ) -> str: """ Turn the data into a dict and then dump to JSON. @@ -256,7 +256,9 @@ def as_json( return as_json.encode(data, default=default, indent=indent, **kwargs) - def json(self, default: t.Callable[[t.Any], t.Any] = None, indent: t.Optional[int] = None, **kwargs: t.Any) -> str: + def json( + self, default: t.Callable[[t.Any], t.Any] | None = None, indent: int | None = None, **kwargs: t.Any + ) -> str: # ty: ignore[invalid-method-override] """ Turn the data into a dict and then dump to JSON. """ @@ -267,7 +269,7 @@ def as_list( compact: bool = False, storage_to_dict: bool = False, datetime_to_str: bool = False, - custom_types: list[type] = None, + custom_types: list[type] | None = None, ) -> list[AnyDict]: """ Get the data in a list of dicts. @@ -277,7 +279,7 @@ def as_list( return [_.as_dict() for _ in self.records.values()] - def __getitem__(self, item: int) -> T_MetaInstance: + def __getitem__(self, item: int) -> T_MetaInstance: # ty: ignore[invalid-method-override] """ You can get a specific row by ID from a typedrows by using rows[idx] notation. @@ -324,10 +326,10 @@ def delete(self) -> bool: def join( self, field: "Field | TypedField[t.Any]", - name: str = None, - constraint: Query = None, - fields: list[str | Field] = None, - orderby: t.Optional[str | Field] = None, + name: str | None = None, + constraint: Query | None = None, + fields: list[str | Field] | None = None, + orderby: str | Field | None = None, ) -> T_MetaInstance: """ This can be used to JOIN with some relationships after the initial select. @@ -345,7 +347,7 @@ def export_to_csv_file( quotechar: str = '"', quoting: int = csv.QUOTE_MINIMAL, represent: bool = False, - colnames: list[str] = None, + colnames: list[str] | None = None, write_colnames: bool = True, *args: t.Any, **kwargs: t.Any, @@ -373,7 +375,7 @@ def from_rows( cls, rows: Rows, model: t.Type[T_MetaInstance], - metadata: Metadata = None, + metadata: Metadata | None = None, into: t.Type[_TypedTable] | None = None, init: t.Callable[[_TypedTable, Row], None] | None = None, ) -> "TypedRows[T_MetaInstance]": @@ -497,7 +499,7 @@ def next(self) -> t.Self: if data["current_page"] >= data["max_page"]: raise StopIteration("Final Page") - return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1) + return t.cast(t.Self, self._query_builder.paginate(limit=data["limit"], page=data["current_page"] + 1)) def previous(self) -> t.Self: """ @@ -507,7 +509,7 @@ def previous(self) -> t.Self: if data["current_page"] <= 1: raise StopIteration("First Page") - return self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1) + return t.cast(t.Self, self._query_builder.paginate(limit=data["limit"], page=data["current_page"] - 1)) def as_dict(self, *_: t.Any, **__: t.Any) -> PaginateDict: # type: ignore """ @@ -525,7 +527,7 @@ class TypedSet(pydal.objects.Set): # pragma: no cover This class is not actually used, only 'cast' by TypeDAL.__call__ """ - def count(self, distinct: t.Optional[bool] = None, cache: AnyDict = None) -> int: + def count(self, distinct: bool | None = None, cache: AnyDict | None = None) -> int: """ Count returns an int. """ diff --git a/src/typedal/serializers/typescript.py b/src/typedal/serializers/typescript.py index 39435b4..2b77927 100644 --- a/src/typedal/serializers/typescript.py +++ b/src/typedal/serializers/typescript.py @@ -9,10 +9,13 @@ from configuraptor import Singleton -try: # optional dependency +if t.TYPE_CHECKING: import typtyp -except ImportError: # pragma: no cover - typtyp = None # type: ignore +else: + try: # optional dependency + import typtyp + except ImportError: # pragma: no cover + typtyp = None def is_supported() -> bool: @@ -34,20 +37,22 @@ def __init__(self) -> None: @property def world(self) -> "typtyp.World | None": """Return the shared typtyp world instance, if typtyp is installed.""" + # no `typtyp is None` check: __init__ already stores None in that case, so re-testing + # the import here would only duplicate it - and leave a branch nothing can reach. return self._world def get(self, model: type) -> type[dict[str, t.Any]] | None: """Return the registered TypedDict for a model, or None if absent.""" return self._types.get(model) - def create(self, model: type, fields: dict[str, t.Any] = None, name: str = "") -> type[dict[str, t.Any]]: + def create(self, model: type, fields: dict[str, t.Any] | None = None, name: str = "") -> type[dict[str, t.Any]]: """ Create/register a TypedDict for a model and add it to the shared world. If the world is unavailable (typtyp not installed), registration is local only. """ name = name or model.__name__ - raw_typed_dict = t.TypedDict(name, fields or {}) + raw_typed_dict = t.TypedDict(name, fields or {}) # ty: ignore[invalid-argument-type, mismatched-type-name] typed_dict = t.cast(type[dict[str, t.Any]], raw_typed_dict) self._types[model] = typed_dict self.add_to_world(typed_dict, name=name) diff --git a/src/typedal/tables.py b/src/typedal/tables.py index 4d941d3..0bf325c 100644 --- a/src/typedal/tables.py +++ b/src/typedal/tables.py @@ -15,6 +15,7 @@ import pydal.objects from pydal._globals import DEFAULT +from pydal.helpers.classes import SQLCallableList from .constants import JOIN_OPTIONS from .core import TypeDAL @@ -71,7 +72,7 @@ def reorder_fields( # Start with desired fields, then append the rest new_order.extend(f for f in table._fields if f not in desired) - table._fields = new_order + table._fields = t.cast(SQLCallableList, new_order) class TableMeta(type): @@ -185,6 +186,12 @@ def all(self: t.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]": """ return self.collect() + async def all_async(self: t.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]": + """ + Async twin of `all()`. Thin wrapper: builds on `collect_async()`. + """ + return await self.collect_async() + def get_relationships(self) -> dict[str, Relationship[t.Any]]: """ Return the registered relationships of the current model. @@ -214,6 +221,51 @@ def insert(self: t.Type[T_MetaInstance], **fields: t.Any) -> T_MetaInstance: # it already is an int but mypy doesn't understand that return self(result) + async def _insert_id_async(self: t.Type[T_MetaInstance], **fields: t.Any) -> t.Any: + """ + The insert itself, returning what pydal's own `Table.insert()` returns: the new id. + + Mirrors pydal's `Table.insert()` (objects.py): the field normalization + (`_fields_and_values_for_insert`) and `_before_insert`/`_after_insert` hooks stay + exactly as they are (pure/sync), only the adapter-level execute step + (`table._db.insert_async(...)`) is async. + + Split out from `insert_async()` because turning that id into a model instance costs a + second query: `bulk_insert_async()` wants the ids only and does that lookup once for + the whole batch, not once per row. + """ + table = self._ensure_table_defined() + require_permission(self._permissions, "insert") + + row = table._fields_and_values_for_insert(fields) + if any(f(row) for f in table._before_insert): + return 0 + + result = await table._db.insert_async(table, row.op_values()) + if result and table._after_insert: + for f in table._after_insert: + f(row, result) + + return result + + async def insert_async(self: t.Type[T_MetaInstance], **fields: t.Any) -> T_MetaInstance: + """ + Async twin of `insert()`. + """ + result = await self._insert_id_async(**fields) + + if not isinstance(result, int) or not result: + # a `_before_insert` hook that blocked the insert (0), or a table with a custom + # `_primarykey`, whose id is a dict - `self(...)` answers None for both, as sync does. + return self(result) + + table = self._ensure_table_defined() + # NOT `self(result)`: that is pydal's *synchronous* `Table.__call__` -> `db(...).select()`, + # which would run a blocking query on the event loop - on pydal's sync connection, no + # less, so it also reads from a different transaction than the insert just wrote in. + row = await self.where(table._id == result).first_async() + return t.cast(T_MetaInstance, row) + def _insert(self, **fields: t.Any) -> str: table = self._ensure_table_defined() @@ -228,9 +280,27 @@ def bulk_insert(self: t.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRow result = table.bulk_insert(items) return self.where(lambda row: row.id.belongs(result)).collect() + async def bulk_insert_async(self: t.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]": + """ + Async twin of `bulk_insert()`. + + pydal's `Table.bulk_insert()` (objects.py) only exists to hand the whole batch + to `adapter.bulk_insert()`, which for every backend TypeDAL supports asynchronously is + itself a loop over `insert()` - so looping the insert here loses nothing and keeps + the hook/normalization dance in one place. + + `_insert_id_async()` rather than `insert_async()`: the ids are all this needs, and the + `collect_async()` below already fetches every inserted row in one query. + """ + self._ensure_table_defined() + + ids = [await self._insert_id_async(**item) for item in items] + + return await self.where(lambda row: row.id.belongs(ids)).collect_async() + def update_or_insert( self: t.Type[T_MetaInstance], - query: T_Query | AnyDict = DEFAULT, + query: T_Query | AnyDict | t.Callable[[], None] = DEFAULT, **values: t.Any, ) -> T_MetaInstance: """ @@ -253,6 +323,59 @@ def update_or_insert( record.update_record(**values) return self(record) + async def update_or_insert_async( + self: t.Type[T_MetaInstance], + query: T_Query | AnyDict | t.Callable[[], None] = DEFAULT, + **values: t.Any, + ) -> T_MetaInstance: + """ + Async twin of `update_or_insert()`. + + The sync version leans on pydal's `table(...)` call syntax for the lookup, which is a + synchronous select; `_lookup_query()` turns the same three input shapes into a plain + Query so the lookup can go through `first_async()` instead. + """ + lookup = self._lookup_query(query, values) + + record = await QueryBuilder(self).where(lookup).first_async() if lookup is not None else None + + if not record: + return await self.insert_async(**values) + + return await record.update_record_async(**values) + + def _lookup_query( + self: t.Type[T_MetaInstance], + query: T_Query | AnyDict | t.Callable[[], None] | None, + values: AnyDict, + ) -> Query | None: + """ + Turn `update_or_insert`'s three input shapes (DEFAULT / dict / Query) into one Query. + + Mirrors pydal's `Table.update_or_insert()` (objects.py): no query means + "match on the values you were going to write", a dict means "match on these fields". + + `None` means "no lookup at all, go straight to the insert" - the caller's cue, not a + failure. `T_Query` includes `None` and `bool`, and pydal's `Table.__call__` + (objects.py) answers anything that is not a Query or a digit-like id with "no + record", which is exactly what makes the sync `update_or_insert(None, ...)` insert. + Handing those to `QueryBuilder.where()` instead would raise ValueError. + """ + table = self._ensure_table_defined() + + if query is not DEFAULT and not isinstance(query, dict): + is_query = isinstance(query, (pydal.objects.Query, pydal.objects.Expression)) + return t.cast(Query, query) if is_query else None + + criteria = values if query is DEFAULT else t.cast(AnyDict, query) + + result = None + for key, value in criteria.items(): + condition = table[key] == value + result = condition if result is None else (result & condition) + + return t.cast(Query, result) + def validate_and_insert( self: t.Type[T_MetaInstance], **fields: t.Any, @@ -270,6 +393,25 @@ def validate_and_insert( else: return None, result.get("errors") + async def validate_and_insert_async( + self: t.Type[T_MetaInstance], + **fields: t.Any, + ) -> tuple[t.Optional[T_MetaInstance], t.Optional[dict[str, str]]]: + """ + Async twin of `validate_and_insert()`. + + Mirrors pydal's `Table.validate_and_insert()` (objects.py): `_validate_fields()` + is pure (no I/O), so only the insert step needs an async twin. + """ + table = self._ensure_table_defined() + require_permission(self._permissions, "insert") + + errors, new_fields = table._validate_fields(fields) + if errors: + return None, errors + + return await self.insert_async(**new_fields), None + def validate_and_update( self: t.Type[T_MetaInstance], query: Query, @@ -293,6 +435,32 @@ def validate_and_update( # update on query without result (shouldnt happen) return None, None + async def validate_and_update_async( + self: t.Type[T_MetaInstance], + query: Query, + **fields: t.Any, + ) -> tuple[t.Optional[T_MetaInstance], t.Optional[dict[str, str]]]: + """ + Async twin of `validate_and_update()`. + + Mirrors pydal's `Table.validate_and_update()` (objects.py): fetch the record, + validate against it (pure), then update. Both DB steps go through the async path. + """ + table = self._ensure_table_defined() + require_permission(self._permissions, "update") + + record = await QueryBuilder(self).where(query).first_async() + + errors, new_fields = table._validate_fields(fields, record._row if record else None) + if errors: + return None, errors + + if not record: # pragma: no cover + # update on query without result (shouldnt happen) + return None, None + + return await record.update_record_async(**new_fields), None + def validate_and_update_or_insert( self: t.Type[T_MetaInstance], query: Query, @@ -321,6 +489,19 @@ def validate_and_update_or_insert( # update on query without result (shouldnt happen) return None, None + async def validate_and_update_or_insert_async( + self: t.Type[T_MetaInstance], + query: Query, + **fields: t.Any, + ) -> tuple[t.Optional[T_MetaInstance], t.Optional[dict[str, str]]]: + """ + Async twin of `validate_and_update_or_insert()`. + """ + if await QueryBuilder(self).where(query).exists_async(): + return await self.validate_and_update_async(query, **fields) + + return await self.validate_and_insert_async(**fields) + def select(self: t.Type[T_MetaInstance], *a: t.Any, **kw: t.Any) -> "QueryBuilder[T_MetaInstance]": """ See QueryBuilder.select! @@ -339,18 +520,45 @@ def column[T: t.Any, T_MetaInstance: _TypedTable]( """ return QueryBuilder(self).select(field, **options).execute().column(field) + async def column_async[T: t.Any, T_MetaInstance: _TypedTable]( + self: t.Type[T_MetaInstance], + field: T | TypedField[T], + **options: t.Unpack[SelectKwargs], + ) -> list[T]: + """ + See QueryBuilder.column_async! + """ + return await QueryBuilder(self).column_async(field, **options) + def paginate(self: t.Type[T_MetaInstance], limit: int, page: int = 1) -> "PaginatedRows[T_MetaInstance]": """ See QueryBuilder.paginate! """ return QueryBuilder(self).paginate(limit=limit, page=page) + async def paginate_async( + self: t.Type[T_MetaInstance], limit: int, page: int = 1 + ) -> "PaginatedRows[T_MetaInstance]": + """ + See QueryBuilder.paginate_async! + """ + return await QueryBuilder(self).paginate_async(limit=limit, page=page) + def chunk(self: t.Type[T_MetaInstance], chunk_size: int) -> t.Generator["TypedRows[T_MetaInstance]", t.Any, None]: """ See QueryBuilder.chunk! """ return QueryBuilder(self).chunk(chunk_size) + async def chunk_async( + self: t.Type[T_MetaInstance], chunk_size: int + ) -> t.AsyncGenerator["TypedRows[T_MetaInstance]", None]: + """ + See QueryBuilder.chunk_async! + """ + async for rows in QueryBuilder(self).chunk_async(chunk_size): + yield rows + def where(self: t.Type[T_MetaInstance], *a: t.Any, **kw: t.Any) -> "QueryBuilder[T_MetaInstance]": """ See QueryBuilder.where! @@ -395,24 +603,48 @@ def count(self: t.Type[T_MetaInstance]) -> int: """ return QueryBuilder(self).count() + async def count_async(self: t.Type[T_MetaInstance]) -> int: + """ + See QueryBuilder.count_async! + """ + return await QueryBuilder(self).count_async() + def exists(self: t.Type[T_MetaInstance]) -> bool: """ See QueryBuilder.exists! """ return QueryBuilder(self).exists() + async def exists_async(self: t.Type[T_MetaInstance]) -> bool: + """ + See QueryBuilder.exists_async! + """ + return await QueryBuilder(self).exists_async() + def first(self: t.Type[T_MetaInstance]) -> T_MetaInstance | None: """ See QueryBuilder.first! """ return QueryBuilder(self).first() + async def first_async(self: t.Type[T_MetaInstance]) -> T_MetaInstance | None: + """ + See QueryBuilder.first_async! + """ + return await QueryBuilder(self).first_async() + def first_or_fail(self: t.Type[T_MetaInstance]) -> T_MetaInstance: """ See QueryBuilder.first_or_fail! """ return QueryBuilder(self).first_or_fail() + async def first_or_fail_async(self: t.Type[T_MetaInstance]) -> T_MetaInstance: + """ + See QueryBuilder.first_or_fail_async! + """ + return await QueryBuilder(self).first_or_fail_async() + def join( self: t.Type[T_MetaInstance], *fields: str | t.Type[TypedTable] | Relationship[t.Any], @@ -432,6 +664,12 @@ def collect(self: t.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T """ return QueryBuilder(self).collect(verbose=verbose) + async def collect_async(self: t.Type[T_MetaInstance], verbose: bool = False) -> "TypedRows[T_MetaInstance]": + """ + See QueryBuilder.collect_async! + """ + return await QueryBuilder(self).collect_async(verbose=verbose) + def collect_into[T_Into: _TypedTable]( self: t.Type[_TypedTable], into: t.Type[T_Into], @@ -443,6 +681,17 @@ def collect_into[T_Into: _TypedTable]( """ return QueryBuilder(self).collect_into(into=into, verbose=verbose, init=init) + async def collect_into_async[T_Into: _TypedTable]( + self: t.Type[_TypedTable], + into: t.Type[T_Into], + verbose: bool = False, + init: t.Callable[[T_Into, Row], None] | None = None, + ) -> "TypedRows[T_Into]": + """ + See QueryBuilder.collect_into_async! + """ + return await QueryBuilder(self).collect_into_async(into=into, verbose=verbose, init=init) + @property def ALL(cls) -> pydal.objects.SQLALL: """ @@ -492,11 +741,11 @@ def drop_index(self, name: str, if_exists: bool = False) -> bool: def import_from_csv_file( self, csvfile: t.TextIO, - id_map: dict[str, str] = None, + id_map: dict[str, str] | None = None, null: t.Any = "", unique: str = "uuid", - id_offset: dict[str, int] = None, # id_offset used only when id_map is None - transform: t.Callable[[dict[t.Any, t.Any]], dict[t.Any, t.Any]] = None, + id_offset: dict[str, int] | None = None, # id_offset used only when id_map is None + transform: t.Callable[[dict[t.Any, t.Any]], dict[t.Any, t.Any]] | None = None, validate: bool = False, encoding: str = "utf-8", delimiter: str = ",", @@ -703,6 +952,7 @@ def reorder_fields(cls, *fields: str | Field | TypedField[t.Any], keep_others: b - True (default): keep other fields at the end, in their original order. - False: remove other fields (only keep what's specified). """ + assert cls._table is not None, "TypedTable.reorder_fields() requires a bound table" return reorder_fields(cls._table, fields, keep_others=keep_others) @@ -730,6 +980,7 @@ class _TypedTable(metaclass=TableMeta): _after_update: list[t.Callable[[Set, t.Self], t.Optional[bool]] | t.Callable[[Set, OpRow], t.Optional[bool]]] _before_delete: list[t.Callable[[Set], t.Optional[bool]]] _after_delete: list[t.Callable[[Set], t.Optional[bool]]] + _row: Row | None _rows: tuple[Row, ...] _with: list[str] @@ -770,6 +1021,10 @@ def update_record(self: t.Self, **fields: t.Any) -> t.Self: # Declared here for generic update flows; real behavior is implemented in TypedTable. raise NotImplementedError # pragma: no cover + async def update_record_async(self: t.Self, **fields: t.Any) -> t.Self: + # Declared here for generic async update flows; real behavior is implemented in TypedTable. + raise NotImplementedError # pragma: no cover + def as_dict(self, *args: t.Any, **kwargs: t.Any) -> AnyDict: # Broad signature keeps class/instance serialization overrides LSP-compatible. raise NotImplementedError # pragma: no cover @@ -934,7 +1189,7 @@ def _setup_instance_methods(self) -> None: def __new__( cls, - row_or_id: t.Union[Row, Query, pydal.objects.Set, int, str, None, "TypedTable"] = None, + row_or_id: t.Union[Row, Query, pydal.objects.Set, int, str, "TypedTable", None] = None, **filters: t.Any, ) -> t.Self: """ @@ -965,7 +1220,7 @@ def __new__( if not row: return None # type: ignore - inst._row = row + inst._row = t.cast(Row, row) if hasattr(row, "id"): inst.__dict__.update(row) @@ -984,7 +1239,7 @@ def __iter__(self) -> t.Generator[t.Any, None, None]: row = self._ensure_matching_row() yield from iter(row) - def __getitem__(self, item: str) -> t.Any: + def __getitem__(self, item: str) -> t.Any: # ty: ignore[invalid-method-override] """ Allows dictionary notation to get columns. """ @@ -1255,8 +1510,8 @@ def asdict_method(obj: t.Any) -> t.Any: # pragma: no cover def _as_json( self, - default: t.Callable[[t.Any], t.Any] = None, - indent: t.Optional[int] = None, + default: t.Callable[[t.Any], t.Any] | None = None, + indent: int | None = None, **kwargs: t.Any, ) -> str: data = self._as_dict() @@ -1294,6 +1549,16 @@ def update(cls: t.Type[T_MetaInstance], query: Query, **fields: t.Any) -> T_Meta else: return None + @classmethod + async def update_async(cls: t.Type[T_MetaInstance], query: Query, **fields: t.Any) -> T_MetaInstance | None: + """ + Async twin of `update()`. Thin wrapper: builds on `update_record_async()`. + """ + if record := await QueryBuilder(cls).where(query).first_async(): + return await record.update_record_async(**fields) + else: + return None + def _update(self: T_MetaInstance, **fields: t.Any) -> T_MetaInstance: require_permission(getattr(self, "_permissions", None), "update") row = self._ensure_matching_row() @@ -1316,6 +1581,36 @@ def update_record(self: T_MetaInstance, **fields: t.Any) -> T_MetaInstance: # p """ return self._update_record(**fields) + async def update_record_async(self: T_MetaInstance, **fields: t.Any) -> T_MetaInstance: + """ + Async twin of `update_record()`. + + Mirrors pydal's `RecordUpdater` (helpers/classes.py): drop anything that isn't a + writable column of this table, update by primary key, then mirror the new values onto the + in-memory row/instance - `_update()` does that last part for both the sync and async path. + + Including `ignore_common_filters=True`, which `RecordUpdater` passes (classes.py): + a record you already hold must always be writable back, even when the table has a + common filter that excludes it - a soft-deleted row, say. Without it `adapter._update()` + re-applies that filter (adapters/base.py via `use_common_filters`) and the + update silently matches zero rows. + """ + require_permission(getattr(self, "_permissions", None), "update") + row = self._ensure_matching_row() + cls = type(self) + table = cls._ensure_table_defined() + + new_fields = {k: v for k, v in fields.items() if k in table.fields and table[k].type != "id"} + + query = t.cast(Query, table._id == row[table._id.name]) + # what `db(query, ignore_common_filters=True)` does under the hood (objects.py); + # set on the Query itself because that object is what reaches `adapter._update()`: + query.ignore_common_filters = True + + await QueryBuilder(cls, query).update_async(**new_fields) + + return self._update(**new_fields) + def _delete_record(self) -> int: """ Actual logic in `pydal.helpers.classes.RecordDeleter`. @@ -1338,6 +1633,27 @@ def delete_record(self) -> int: # pragma: no cover """ return self._delete_record() + async def delete_record_async(self) -> int: + """ + Async twin of `delete_record()`. + + Mirrors pydal's `RecordDeleter` (helpers/classes.py) plus `_delete_record()`'s + own bookkeeping: the instance is emptied afterwards, since the row is no more. + """ + require_permission(getattr(self, "_permissions", None), "delete") + row = self._ensure_matching_row() + cls = type(self) + table = cls._ensure_table_defined() + + deleted = await QueryBuilder(cls).where(table._id == row[table._id.name]).delete_async() + + self.__dict__ = {} # empty self, since row is no more. + self._row = None # just to be sure + self._setup_instance_methods() + # ^ instance methods might've been deleted by emptying dict, + # but we still want .as_dict to show an error, not the table's as_dict. + return len(deleted) + # __del__ is also called on the end of a scope so don't remove records on every del!! # pickling: @@ -1382,9 +1698,9 @@ def _sql(cls) -> str: except ImportError as e: # pragma: no cover raise RuntimeError("Can not generate SQL without the 'migration' extra or `pydal2sql` installed!") from e - return pydal2sql.generate_sql(cls) + return pydal2sql.generate_sql(cls) # ty: ignore[invalid-argument-type] - def render(self, fields: list[Field] = None, compact: bool = False) -> t.Self: + def render(self, fields: list[Field] | None = None, compact: bool = False) -> t.Self: """ Renders a copy of the object with potentially modified values. @@ -1395,6 +1711,9 @@ def render(self, fields: list[Field] = None, compact: bool = False) -> t.Self: Returns: A copy of the object with potentially modified values. """ + assert self._db is not None, "TypedTable.render() requires a bound database" + assert self._table is not None, "TypedTable.render() requires a bound table" + assert self._relationships is not None, "TypedTable.render() requires relationship metadata" row = copy.deepcopy(self) keys = list(row) if not fields: @@ -1416,6 +1735,7 @@ def render(self, fields: list[Field] = None, compact: bool = False) -> t.Self: relation_table = relation.table if isinstance(relation_table, str): relation_table = self._db[relation_table] + assert relation_table is not None, f"Relationship {relation_name!r} has no table" relation_row = row[relation_name] @@ -1428,7 +1748,7 @@ def render(self, fields: list[Field] = None, compact: bool = False) -> t.Self: for related_og in relation_row: related = copy.deepcopy(related_og) for fieldname in related: - field = relation_table[fieldname] + field = relation_table[fieldname] # ty: ignore[not-subscriptable] related[field.name] = self._db.represent( "rows_render", field, @@ -1440,8 +1760,9 @@ def render(self, fields: list[Field] = None, compact: bool = False) -> t.Self: row[relation_name] = combined else: # 1 row + assert relation_row is not None, f"Relationship {relation_name!r} has no row" for fieldname in relation_row: - field = relation_table[fieldname] + field = relation_table[fieldname] # ty: ignore[not-subscriptable] row[relation_name][fieldname] = self._db.represent( "rows_render", field, diff --git a/src/typedal/types.py b/src/typedal/types.py index e3cc6d3..4025372 100644 --- a/src/typedal/types.py +++ b/src/typedal/types.py @@ -27,7 +27,7 @@ try: from string.templatelib import Template as TemplateAlias except ImportError: - TemplateAlias: t.TypeAlias = str # type: ignore + TemplateAlias: t.TypeAlias = str # Internal references if t.TYPE_CHECKING: @@ -71,7 +71,7 @@ def merge_permissions(*permission_sets: Permissions | None) -> Permissions: for key in permission_types: if key in permission_set: - merged[key] = merged[key] and bool(permission_set[key]) # type: ignore + merged[key] = merged[key] and bool(permission_set[key]) return t.cast(Permissions, merged) @@ -404,10 +404,10 @@ class DefineKwargs(t.TypedDict, total=False): "Table", Query, bool, - None, "TypedTable", t.Type["TypedTable"], Expression, + None, ] type T_Field = t.Union["TypedField[t.Any]", "Table", t.Type["TypedTable"]] diff --git a/src/typedal/web2py_py4web_shared.py b/src/typedal/web2py_py4web_shared.py index 91c89bb..6fd1a88 100644 --- a/src/typedal/web2py_py4web_shared.py +++ b/src/typedal/web2py_py4web_shared.py @@ -3,11 +3,13 @@ """ import datetime as dt +import typing as t from pydal.validators import CRYPT, IS_EMAIL, IS_NOT_EMPTY, IS_NOT_IN_DB, IS_STRONG from . import TypeDAL, TypedField, TypedTable from .fields import PasswordField +from .types import Validator class AuthUser(TypedTable): @@ -35,10 +37,11 @@ def __on_define__(cls, db: TypeDAL) -> None: """ super().__on_define__(db) - cls.email.requires = [ + requires = [ IS_EMAIL(), IS_NOT_IN_DB( db, "auth_user.email", ), ] + cls.email.requires = t.cast(list[Validator], requires) diff --git a/tests/conftest.py b/tests/conftest.py index 8286809..3879d30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,5 +28,14 @@ def dal_psql_uri(psql) -> str: @pytest.fixture def dal_psql(dal_psql_uri: str): + # function-scoped, so this runs once per test - which makes closing it mandatory rather + # than tidy. Without the close each test leaves a Postgres connection open (idle in + # transaction, since migrate=True runs DDL on it), and the container's default ceiling of + # 100 is reached partway through the suite: everything from then on fails to connect with + # `FATAL: sorry, too many clients already`. with tempfile.TemporaryDirectory() as d: - yield TypeDAL(dal_psql_uri, attempts=1, migrate=True, enable_typedal_caching=False, folder=d) + db = TypeDAL(dal_psql_uri, attempts=1, migrate=True, enable_typedal_caching=False, folder=d) + try: + yield db + finally: + db.close() diff --git a/tests/test_async_execution.py b/tests/test_async_execution.py new file mode 100644 index 0000000..d368e24 --- /dev/null +++ b/tests/test_async_execution.py @@ -0,0 +1,2380 @@ +""" +Test-first spec for TypeDAL's async execution path. + +Scope is Postgres AND SQLite together, not sequenced - `db_async` is parametrized over both +backends so every test below runs against each, proving the same async surface works +identically rather than "works for Postgres, TODO for SQLite". + +Covers two concrete Postgres divergence points found while building this: + - jsonb -> dict (pydal's Postgres parser expects the driver to have already decoded it) + - decimal(10,2) -> Decimal +and the actual point of the exercise: the event loop is not blocked while the query runs. +""" + +import asyncio +import collections +import contextlib +import signal +import sqlite3 +import tempfile +import time +import typing as t +from decimal import Decimal +from pathlib import Path + +import pydal.objects +import pytest +import pytest_asyncio + +from src.typedal import TypeDAL, TypedField, TypedTable +from src.typedal.async_execution import ( + ASYNC_POOL_FACTORIES, + AsyncPoolManager, + ConcurrentTransactionError, + PostgresAsyncPool, + TransactionBoundaryError, + TransactionSplitError, + open_sqlite_async_connection, + postgres_lastrowid_async, +) +from src.typedal.fields import DecimalField, JSONField +from src.typedal.query_builder import QueryBuilder + + +@contextlib.asynccontextmanager +async def _postgres_db(dal_psql: TypeDAL) -> t.AsyncIterator[TypeDAL]: + try: + yield dal_psql + finally: + await dal_psql.close_async() + + +@contextlib.asynccontextmanager +async def _sqlite_db(dal_psql: TypeDAL | None = None) -> t.AsyncIterator[TypeDAL]: + # `dal_psql` is unused and optional so this doubles as the `db_sqlite_memory` fixture's + # factory: the `sqlite:memory`-only tests below have no reason to start a Postgres container. + with tempfile.TemporaryDirectory() as d: + db = TypeDAL("sqlite:memory", enable_typedal_caching=False, folder=d) + try: + yield db + finally: + await db.close_async() + db.close() + + +@contextlib.asynccontextmanager +async def _sqlite_file_db(dal_psql: TypeDAL) -> t.AsyncIterator[TypeDAL]: + """ + A file-backed SQLite database, which is a materially different async backend from + `sqlite:memory` and not a redundant copy of it. + + `sqlite:memory` reaches a second connection only through shared-cache mode, which refuses a + concurrent writer with SQLITE_LOCKED, so its async path is one shared connection + (`SqliteAsyncConnection`) that turns a second task away. A file has a path two connections + can both open, so it gets `SqliteAsyncPool` and a connection per task instead. Every + transaction-boundary claim differs between the two, and without this parametrization the + per-task SQLite code is never executed by the suite at all. + """ + with tempfile.TemporaryDirectory() as d: + db = TypeDAL(f"sqlite://{Path(d) / 'async.db'}", enable_typedal_caching=False, folder=d) + try: + yield db + finally: + await db.close_async() + db.close() + + +# One factory per backend the async execution path targets. +_ASYNC_DB_FACTORIES: dict[str, t.Callable[[TypeDAL], t.AsyncContextManager[TypeDAL]]] = { + "postgres": _postgres_db, + "sqlite": _sqlite_db, + "sqlite-file": _sqlite_file_db, +} + + +@pytest_asyncio.fixture(params=list(_ASYNC_DB_FACTORIES)) +async def db_async(request: pytest.FixtureRequest, dal_psql: TypeDAL) -> t.AsyncIterator[TypeDAL]: + """ + A `TypeDAL` instance for each backend the async execution path targets, with a guaranteed- + closed async connection pool afterwards. + + Without the teardown, a lazily-opened async pool/connection outlives the test's event loop + (pytest-asyncio gives each test function its own loop by default) and the *next* test hangs + trying to use pool internals (locks/tasks) bound to an already-closed loop. + """ + factory = _ASYNC_DB_FACTORIES[request.param] + async with factory(dal_psql) as db: + yield db + + +@pytest_asyncio.fixture +async def db_sqlite_memory() -> t.AsyncIterator[TypeDAL]: + """ + A `sqlite:memory` `TypeDAL`, for the claims that only exist on `SqliteAsyncConnection`. + + Not a slice of `db_async`: the tests using this are about the one-connection backend + specifically - a second task being refused, and what the single shared transaction does + when its owner never ends it - which has no counterpart on the two per-task backends. + """ + async with _sqlite_db() as db: + yield db + + +async def _abandon_async_transaction(table: t.Any) -> None: + """Leave a transaction open by inserting a row in a task that ends without settling.""" + async def insert_and_abandon() -> None: + await table.insert_async(name="abandoned") + + await asyncio.create_task(insert_and_abandon()) + await asyncio.sleep(0) + await asyncio.sleep(0) + + +async def _set_finished_owner(pool: t.Any) -> t.Any: + """Make `pool` look like a task finished while still owning its transaction.""" + finished = asyncio.create_task(asyncio.sleep(0)) + await finished + pool._owner = finished + return finished + + +@pytest.mark.asyncio +async def test_collect_async_matches_sync_collect(db_async: TypeDAL): + """The core parity claim: async-executed rows must equal sync-executed rows, field for field.""" + db = db_async + + @db.define() + class AsyncThingParity(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingParity.insert(name="widget", qty=3) + AsyncThingParity.insert(name="gadget", qty=7) + db.commit() + + sync_rows = AsyncThingParity.where(AsyncThingParity.qty > 0).collect() + async_rows = await AsyncThingParity.where(AsyncThingParity.qty > 0).collect_async() + + assert len(async_rows) == len(sync_rows) == 2 + + sync_by_id = {row.id: row for row in sync_rows} + async_by_id = {row.id: row for row in async_rows} + assert sync_by_id.keys() == async_by_id.keys() + + for row_id, sync_row in sync_by_id.items(): + async_row = async_by_id[row_id] + assert async_row.name == sync_row.name + assert async_row.qty == sync_row.qty + + +@pytest.mark.asyncio +async def test_collect_async_preserves_types(db_async: TypeDAL): + """The two divergence points the spike actually found: decimal and jsonb.""" + db = db_async + + @db.define() + class AsyncThingTypes(TypedTable): + name: TypedField[str] + price = DecimalField(10, 2) + meta = JSONField() + + AsyncThingTypes.insert(name="widget", price=Decimal("19.99"), meta={"a": 1, "b": [1, 2, 3]}) + db.commit() + + rows = await AsyncThingTypes.where(AsyncThingTypes.name == "widget").collect_async() + row = rows.first() + + assert isinstance(row.price, Decimal) + assert row.price == Decimal("19.99") + + assert isinstance(row.meta, dict) # not a raw jsonb/json string + assert row.meta == {"a": 1, "b": [1, 2, 3]} + + +@pytest.mark.asyncio +async def test_count_async_matches_sync_count(db_async: TypeDAL): + """count_async must return the same count as the sync count().""" + db = db_async + + @db.define() + class AsyncThingCount(TypedTable): + qty: TypedField[int] + + AsyncThingCount.insert(qty=1) + AsyncThingCount.insert(qty=2) + AsyncThingCount.insert(qty=3) + db.commit() + + sync_count = AsyncThingCount.where(AsyncThingCount.qty > 1).count() + async_count = await AsyncThingCount.where(AsyncThingCount.qty > 1).count_async() + + assert async_count == sync_count == 2 + + +@pytest.mark.asyncio +async def test_insert_async_matches_sync_insert(db_async: TypeDAL): + """insert_async must return a usable id, and the row must actually be committed and visible.""" + db = db_async + + @db.define() + class AsyncThingInsert(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + new_id = await AsyncThingInsert.insert_async(name="widget", qty=5) + await db.commit_async() + + assert int(new_id) > 0 + + row = AsyncThingInsert.where(AsyncThingInsert.id == int(new_id)).first() + assert row is not None + assert row.name == "widget" + assert row.qty == 5 + + +@pytest.mark.asyncio +async def test_update_async_matches_sync_update(db_async: TypeDAL): + """update_async must update the same rows as the sync update() and return matching ids.""" + db = db_async + + @db.define() + class AsyncThingUpdate(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingUpdate.insert(name="widget", qty=1) + AsyncThingUpdate.insert(name="gadget", qty=2) + db.commit() + + updated_ids = await AsyncThingUpdate.where(AsyncThingUpdate.qty > 0).update_async(qty=99) + await db.commit_async() + + assert len(updated_ids) == 2 + + rows = AsyncThingUpdate.where(AsyncThingUpdate.qty == 99).collect() + assert len(rows) == 2 + + +@pytest.mark.asyncio +async def test_delete_async_matches_sync_delete(db_async: TypeDAL): + """delete_async must delete the same rows as the sync delete() and return matching ids.""" + db = db_async + + @db.define() + class AsyncThingDelete(TypedTable): + qty: TypedField[int] + + AsyncThingDelete.insert(qty=1) + AsyncThingDelete.insert(qty=2) + db.commit() + + deleted_ids = await AsyncThingDelete.where(AsyncThingDelete.qty > 0).delete_async() + await db.commit_async() + + assert len(deleted_ids) == 2 + + remaining = AsyncThingDelete.where(AsyncThingDelete.qty > 0).count() + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_executesql_async_matches_sync_executesql(db_async: TypeDAL): + """executesql_async must return the same raw rows as the sync executesql().""" + db = db_async + + @db.define() + class AsyncThingRaw(TypedTable): + qty: TypedField[int] + + AsyncThingRaw.insert(qty=1) + AsyncThingRaw.insert(qty=2) + db.commit() + + query = f"SELECT qty FROM {AsyncThingRaw._table._rname} ORDER BY qty;" + + sync_rows = db.executesql(query) + async_rows = await db.executesql_async(query) + + assert list(async_rows) == list(sync_rows) == [(1,), (2,)] + + +@pytest.mark.asyncio +async def test_collect_async_with_relationships_matches_sync(db_async: TypeDAL): + """ + Relationships/joins must load through the async path too. Nothing on that path executes a + second query: the joins are in the single query built by `_before_query()` and + `_collect_with_relationships()` only maps already-fetched rows, so async parity is expected. + """ + db = db_async + + @db.define() + class AsyncThingRelOther(TypedTable): + name: TypedField[str] + + @db.define() + class AsyncThingRelMain(TypedTable): + name: TypedField[str] + other: AsyncThingRelOther + + other_id = AsyncThingRelOther.insert(name="parent") + AsyncThingRelMain.insert(name="child", other=other_id) + db.commit() + + sync_rows = AsyncThingRelMain.join("other").collect() + async_rows = await AsyncThingRelMain.join("other").collect_async() + + assert len(async_rows) == len(sync_rows) == 1 + + sync_row = sync_rows.first() + async_row = async_rows.first() + assert async_row.name == sync_row.name == "child" + assert async_row.other.name == sync_row.other.name == "parent" + + # and with a limitby, which routes through `_apply_limitby_optimization()`'s id-subquery: + paginated = await AsyncThingRelMain.join("other").paginate_async(limit=1, page=1) + assert len(paginated) == 1 + assert paginated.first().other.name == "parent" + + +@pytest.mark.asyncio +async def test_all_async_matches_sync_all(db_async: TypeDAL): + """all_async must return the same rows as the sync all().""" + db = db_async + + @db.define() + class AsyncThingAll(TypedTable): + qty: TypedField[int] + + AsyncThingAll.insert(qty=1) + AsyncThingAll.insert(qty=2) + db.commit() + + sync_rows = AsyncThingAll.all() + async_rows = await AsyncThingAll.all_async() + + assert len(async_rows) == len(sync_rows) == 2 + + +@pytest.mark.asyncio +async def test_exists_async_matches_sync_exists(db_async: TypeDAL): + """exists_async (QueryBuilder and the TypedTable shortcut) must match the sync exists().""" + db = db_async + + @db.define() + class AsyncThingExists(TypedTable): + qty: TypedField[int] + + assert not await AsyncThingExists.where(AsyncThingExists.qty > 0).exists_async() + assert not await AsyncThingExists.exists_async() + + AsyncThingExists.insert(qty=1) + db.commit() + + assert AsyncThingExists.where(AsyncThingExists.qty > 0).exists() is True + assert await AsyncThingExists.where(AsyncThingExists.qty > 0).exists_async() is True + assert await AsyncThingExists.exists_async() is True + + +@pytest.mark.asyncio +async def test_first_async_and_first_or_fail_async_match_sync(db_async: TypeDAL): + """first_async/first_or_fail_async (QueryBuilder and TypedTable shortcuts) must match sync.""" + db = db_async + + @db.define() + class AsyncThingFirst(TypedTable): + qty: TypedField[int] + + assert await AsyncThingFirst.where(AsyncThingFirst.qty > 0).first_async() is None + with pytest.raises(ValueError): + await AsyncThingFirst.where(AsyncThingFirst.qty > 0).first_or_fail_async() + + AsyncThingFirst.insert(qty=5) + db.commit() + + sync_row = AsyncThingFirst.where(AsyncThingFirst.qty > 0).first() + async_row = await AsyncThingFirst.where(AsyncThingFirst.qty > 0).first_async() + assert async_row is not None and sync_row is not None + assert async_row.qty == sync_row.qty == 5 + + async_row_2 = await AsyncThingFirst.where(AsyncThingFirst.qty > 0).first_or_fail_async() + assert async_row_2.qty == 5 + + # TypedTable-level shortcuts (no explicit .where(...)): + async_row_3 = await AsyncThingFirst.first_async() + assert async_row_3 is not None + assert async_row_3.qty == 5 + async_row_4 = await AsyncThingFirst.first_or_fail_async() + assert async_row_4.qty == 5 + + +@pytest.mark.asyncio +async def test_paginate_async_matches_sync_paginate(db_async: TypeDAL): + """paginate_async (QueryBuilder and the TypedTable shortcut) must match sync paginate().""" + db = db_async + + @db.define() + class AsyncThingPaginate(TypedTable): + qty: TypedField[int] + + for i in range(5): + AsyncThingPaginate.insert(qty=i) + db.commit() + + sync_page = AsyncThingPaginate.where(AsyncThingPaginate.qty >= 0).paginate(limit=2, page=2) + async_page = await AsyncThingPaginate.where(AsyncThingPaginate.qty >= 0).paginate_async(limit=2, page=2) + + assert len(async_page) == len(sync_page) == 2 + assert async_page.pagination["current_page"] == sync_page.pagination["current_page"] == 2 + assert async_page.pagination["total_items"] == sync_page.pagination["total_items"] == 5 + + async_page_2 = await AsyncThingPaginate.paginate_async(limit=2, page=1) + assert len(async_page_2) == 2 + + +@pytest.mark.asyncio +async def test_chunk_async_matches_sync_chunk(db_async: TypeDAL): + """chunk_async (QueryBuilder and the TypedTable shortcut) must yield the same chunks as sync chunk().""" + db = db_async + + @db.define() + class AsyncThingChunk(TypedTable): + qty: TypedField[int] + + for i in range(5): + AsyncThingChunk.insert(qty=i) + db.commit() + + sync_chunks = [len(chunk) for chunk in AsyncThingChunk.where(AsyncThingChunk.qty >= 0).chunk(2)] + + async_chunks = [] + async for chunk in AsyncThingChunk.where(AsyncThingChunk.qty >= 0).chunk_async(2): + async_chunks.append(len(chunk)) + + assert async_chunks == sync_chunks == [2, 2, 1] + + async_chunks_2 = [len(chunk) async for chunk in AsyncThingChunk.chunk_async(2)] + assert async_chunks_2 == [2, 2, 1] + + +@pytest.mark.asyncio +async def test_column_async_matches_sync_column(db_async: TypeDAL): + """column_async (QueryBuilder and the TypedTable shortcut) must match sync column().""" + db = db_async + + @db.define() + class AsyncThingColumn(TypedTable): + qty: TypedField[int] + + AsyncThingColumn.insert(qty=1) + AsyncThingColumn.insert(qty=2) + db.commit() + + sync_values = AsyncThingColumn.where(AsyncThingColumn.qty > 0).column(AsyncThingColumn.qty) + async_values = await AsyncThingColumn.where(AsyncThingColumn.qty > 0).column_async(AsyncThingColumn.qty) + + assert sorted(async_values) == sorted(sync_values) == [1, 2] + + async_values_2 = await AsyncThingColumn.column_async(AsyncThingColumn.qty) + assert sorted(async_values_2) == [1, 2] + + +@pytest.mark.asyncio +async def test_collect_into_async_matches_sync_collect_into(db_async: TypeDAL): + """collect_into_async (QueryBuilder and the TypedTable shortcut) must match sync collect_into().""" + db = db_async + + @db.define() + class AsyncThingIntoSource(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + # collect_into reshapes rows from the SAME table into a different Python representation - + # it is not for copying between two distinct tables. These stay undefined (no @db.define()): + # _validate_collect_into_model binds each one to the source's table on first use. + class AsyncThingIntoTargetSync(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + class AsyncThingIntoTargetAsync(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + class AsyncThingIntoTargetAsyncBare(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingIntoSource.insert(name="widget", qty=1) + db.commit() + + sync_rows = AsyncThingIntoSource.where(AsyncThingIntoSource.qty > 0).collect_into(AsyncThingIntoTargetSync) + async_rows = await AsyncThingIntoSource.where(AsyncThingIntoSource.qty > 0).collect_into_async( + AsyncThingIntoTargetAsync, + ) + + assert len(async_rows) == len(sync_rows) == 1 + assert isinstance(async_rows.first(), AsyncThingIntoTargetAsync) + + async_rows_2 = await AsyncThingIntoSource.collect_into_async(AsyncThingIntoTargetAsyncBare) + assert len(async_rows_2) == 1 + + +@pytest.mark.asyncio +async def test_collect_or_fail_async_matches_sync_collect_or_fail(db_async: TypeDAL): + """collect_or_fail_async must match sync collect_or_fail(): rows when present, raise when empty.""" + db = db_async + + @db.define() + class AsyncThingCollectOrFail(TypedTable): + qty: TypedField[int] + + with pytest.raises(ValueError): + await AsyncThingCollectOrFail.where(AsyncThingCollectOrFail.qty > 0).collect_or_fail_async() + + AsyncThingCollectOrFail.insert(qty=1) + db.commit() + + sync_rows = AsyncThingCollectOrFail.where(AsyncThingCollectOrFail.qty > 0).collect_or_fail() + async_rows = await AsyncThingCollectOrFail.where(AsyncThingCollectOrFail.qty > 0).collect_or_fail_async() + + assert len(async_rows) == len(sync_rows) == 1 + + +@pytest.mark.asyncio +async def test_bulk_insert_async_matches_sync_bulk_insert(db_async: TypeDAL): + """bulk_insert_async must insert the same rows as the sync bulk_insert().""" + db = db_async + + @db.define() + class AsyncThingBulkInsert(TypedTable): + qty: TypedField[int] + + rows = await AsyncThingBulkInsert.bulk_insert_async([{"qty": 1}, {"qty": 2}, {"qty": 3}]) + await db.commit_async() + + assert len(rows) == 3 + assert sorted(r.qty for r in rows) == [1, 2, 3] + assert AsyncThingBulkInsert.count() == 3 + + +@pytest.mark.asyncio +async def test_update_or_insert_async_matches_sync(db_async: TypeDAL): + """update_or_insert_async must insert when no match exists, and update when one does.""" + db = db_async + + @db.define() + class AsyncThingUpsert(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + # insert branch: no matching row yet + inserted = await AsyncThingUpsert.update_or_insert_async({"name": "widget"}, name="widget", qty=1) + await db.commit_async() + assert inserted.qty == 1 + assert AsyncThingUpsert.count() == 1 + + # update branch: matching row exists + updated = await AsyncThingUpsert.update_or_insert_async({"name": "widget"}, name="widget", qty=2) + await db.commit_async() + assert updated.qty == 2 + assert AsyncThingUpsert.count() == 1 + + +@pytest.mark.asyncio +async def test_validate_and_insert_async_matches_sync(db_async: TypeDAL): + """validate_and_insert_async must match sync validate_and_insert(): row on success, errors on failure.""" + db = db_async + + @db.define() + class AsyncThingValidateInsert(TypedTable): + qty: TypedField[int] + + row, errors = await AsyncThingValidateInsert.validate_and_insert_async(qty=5) + await db.commit_async() + assert errors is None + assert row is not None + assert row.qty == 5 + + _row, errors = await AsyncThingValidateInsert.validate_and_insert_async(qty="not-a-number") + assert errors is not None + + +@pytest.mark.asyncio +async def test_validate_and_update_async_matches_sync(db_async: TypeDAL): + """validate_and_update_async must match sync validate_and_update(): row on success, errors on failure.""" + db = db_async + + @db.define() + class AsyncThingValidateUpdate(TypedTable): + qty: TypedField[int] + + existing_id = AsyncThingValidateUpdate.insert(qty=1) + db.commit() + + row, errors = await AsyncThingValidateUpdate.validate_and_update_async( + AsyncThingValidateUpdate.id == int(existing_id), + qty=9, + ) + await db.commit_async() + assert errors is None + assert row is not None + assert row.qty == 9 + + _row, errors = await AsyncThingValidateUpdate.validate_and_update_async( + AsyncThingValidateUpdate.id == int(existing_id), + qty="not-a-number", + ) + assert errors is not None + + +@pytest.mark.asyncio +async def test_validate_and_update_or_insert_async_matches_sync(db_async: TypeDAL): + """validate_and_update_or_insert_async must insert when no match exists, update when one does.""" + db = db_async + + @db.define() + class AsyncThingValidateUpsert(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + inserted, errors = await AsyncThingValidateUpsert.validate_and_update_or_insert_async( + AsyncThingValidateUpsert.name == "widget", + name="widget", + qty=1, + ) + await db.commit_async() + assert errors is None + assert inserted.qty == 1 + assert AsyncThingValidateUpsert.count() == 1 + + updated, errors = await AsyncThingValidateUpsert.validate_and_update_or_insert_async( + AsyncThingValidateUpsert.name == "widget", + name="widget", + qty=2, + ) + await db.commit_async() + assert errors is None + assert updated.qty == 2 + assert AsyncThingValidateUpsert.count() == 1 + + +@pytest.mark.asyncio +async def test_classmethod_update_async_matches_sync(db_async: TypeDAL): + """The classmethod update_async(query, **fields) shortcut must match sync update().""" + db = db_async + + @db.define() + class AsyncThingClsUpdate(TypedTable): + qty: TypedField[int] + + existing_id = AsyncThingClsUpdate.insert(qty=1) + db.commit() + + updated = await AsyncThingClsUpdate.update_async(AsyncThingClsUpdate.id == int(existing_id), qty=42) + await db.commit_async() + + assert updated is not None + assert updated.qty == 42 + + +@pytest.mark.asyncio +async def test_update_record_async_and_delete_record_async_match_sync(db_async: TypeDAL): + """Instance-level update_record_async/delete_record_async must match their sync twins.""" + db = db_async + + @db.define() + class AsyncThingRecord(TypedTable): + qty: TypedField[int] + + row_id = AsyncThingRecord.insert(qty=1) + db.commit() + + row = AsyncThingRecord.where(AsyncThingRecord.id == int(row_id)).first() + updated_row = await row.update_record_async(qty=7) + await db.commit_async() + assert updated_row.qty == 7 + + fresh = AsyncThingRecord.where(AsyncThingRecord.id == int(row_id)).first() + assert fresh.qty == 7 + + deleted_count = await fresh.delete_record_async() + await db.commit_async() + assert deleted_count == 1 + assert AsyncThingRecord.count() == 0 + + +@pytest.mark.asyncio +async def test_collect_async_does_not_block_event_loop(db_async: TypeDAL): + """ + The actual point of building this: a query in flight must not stall other coroutines. + A ticker sleeping every 5ms should keep ticking at ~5ms while queries run concurrently; + a blocking implementation would show gaps close to the total query time instead. + """ + db = db_async + + @db.define() + class AsyncThingBlocking(TypedTable): + qty: TypedField[int] + + AsyncThingBlocking.insert(qty=1) + db.commit() + + ticks: list[float] = [] + + async def ticker(): + for _ in range(20): + ticks.append(time.perf_counter()) + await asyncio.sleep(0.005) + + async def repeated_query(): + for _ in range(20): + await AsyncThingBlocking.where(AsyncThingBlocking.qty > 0).collect_async() + + await asyncio.gather(ticker(), repeated_query()) + + gaps = [b - a for a, b in zip(ticks, ticks[1:])] + # generous margin over the 5ms sleep interval; a blocking call would blow well past this + assert max(gaps) < 0.05, f"event loop was blocked: max gap between ticks was {max(gaps) * 1000:.1f}ms" + + +@pytest.mark.asyncio +async def test_insert_async_can_be_rolled_back(db_async: TypeDAL): + """ + (1/3) An `_async` write must leave its transaction open, the way its sync twin does. + + Neither backend does today, and each for its own reason: + + - SQLite: `SqliteAsyncConnection.connection()` (async_execution.py) commits on clean + exit, so the write is durable before `insert_async()` returns. + - Postgres: psycopg_pool's `connection()` applies the same commit-on-success behaviour, + and `PostgresAsyncPool.commit()`/`rollback()` are therefore literally `pass`. + `rollback_async()` is a no-op that reads like transaction control. + + Both are known and documented (see the `PostgresAsyncPool` and `AsyncConnectionPool` + docstrings, and `TypeDAL.commit_async` in core.py). Documented is not the same as safe: a + py4web handler calling `insert_async()` silently falls outside the framework's + rollback-on-error, and gets no signal that it has. + + No concurrency here on purpose. This is a single-coroutine defect, and until it is fixed no + coroutine can hold an open transaction at all - which makes (3/3) unattributable, since it + would fail for this reason no matter how connections are bound. + """ + db = db_async + + @db.define() + class AsyncThingUndoable(TypedTable): + name: TypedField[str] + + db.commit() + + await AsyncThingUndoable.insert_async(name="discard") + await db.rollback_async() + + rows = await AsyncThingUndoable.collect_async() + assert [row.name for row in rows] == [], "rollback_async() did not undo insert_async()" + + # and the sync rollback a framework issues on an unhandled exception must not undo it + # either way round - assert it separately so a fix that only wires up one of the two is + # visible as such. + db.rollback() + assert AsyncThingUndoable.count() == 0, "the write survived both rollbacks" + + +@pytest.mark.asyncio +async def test_crossing_the_sync_async_seam_is_a_loud_error(db_async: TypeDAL): + """ + (2/3) A read must never quietly miss the other connection's uncommitted writes. + + `_async` methods run on a connection opened by `AsyncPoolManager`; sync methods run on + pydal's own, bound to the `THREAD_LOCAL` in pydal's `ConnectionPool`. Those cannot be made + into one connection - pydal drives Postgres with psycopg2 and SQLite with sqlite3, neither + of which can be awaited - so read-your-own-writes across the two paths is not available at + any price. Left alone it failed silently: Postgres returned nothing, SQLite blocked on the + table lock and then raised `database table is locked`. + + This test used to assert cross-visibility outright and closed with "if the split is made + explicit instead, invert this to assert the raised error". That is what happened. + + Warning and continuing was measured before settling on a raise, and does not survive + contact with SQLite: Postgres can return the committed rows and warn, a plain SQLite read + cannot execute at all, and SQLite with `PRAGMA read_uncommitted=1` returns *more* rows than + Postgres - including ones a rollback then deletes. Three answers to identical code, two + silent. See `TransactionSplitError`. + + Both directions asserted, because different machinery guards each and a regression could + hit only one: + + - sync write -> async read: the flag check in `TypeDAL._get_async_pool()`. + - async write -> sync read: `SyncTransactionTracker`, a pydal `ExecutionHandler`, which + sees every statement that reaches the adapter. + + The tail matters most in practice: after committing, the same calls go through. The guard + gates on there being pending work, not on the two paths having been mixed at all - the + latter would make the async path unusable in any handler that also touches pydal. + """ + db = db_async + + @db.define() + class AsyncThingCrossVisibility(TypedTable): + name: TypedField[str] + + db.commit() + + # sync write, not committed -> the async read must refuse rather than silently miss it + AsyncThingCrossVisibility.insert(name="from-sync") + with pytest.raises(TransactionSplitError, match="synchronous connection has uncommitted writes"): + await AsyncThingCrossVisibility.collect_async() + + db.commit() + + # async write, not committed -> the sync read must refuse rather than silently miss it + await AsyncThingCrossVisibility.insert_async(name="from-async") + with pytest.raises(TransactionSplitError, match="async connection has uncommitted writes"): + AsyncThingCrossVisibility.collect() + + # and once both sides are settled, mixing the two paths is ordinary business again + await db.commit_async() + assert sorted(row.name for row in AsyncThingCrossVisibility.collect()) == ["from-async", "from-sync"] + assert sorted(row.name for row in await AsyncThingCrossVisibility.collect_async()) == [ + "from-async", + "from-sync", + ] + + +@pytest.mark.asyncio +async def test_concurrent_coroutines_do_not_share_one_transaction(dal_psql: TypeDAL): + """ + (3/3) The transaction must be bound per task, so two coroutines on one event-loop thread do + not decide each other's commits and rollbacks. + + This is the hazard the issue describes, arriving on the path this package owns. pydal's + `ConnectionPool` binds connection and cursor to a global `THREAD_LOCAL`; under the + threadpool model one thread is one request, so that is the right boundary, but under + `async def` handlers it is not. `_async` methods move off `THREAD_LOCAL`, and this asserts + what they land on instead: `PostgresAsyncPool` pins a checked-out connection to the running + task in a `ContextVar` and holds it until that task ends its own transaction. + + Postgres only, on `dal_psql` rather than the parametrized `db_async`, because it is the only + backend that can run the interleave below at all. The two coroutines have to be inside + separate write transactions simultaneously, and SQLite permits exactly one writer at a time + regardless of how many connections it is given - `sqlite:memory` refuses the second outright + with `ConcurrentTransactionError`, and a file-backed database waits out `busy_timeout` and + then reports `database is locked`. Neither is a defect, and neither can reach the assertion. + + That is a narrower fixture, not a skip: the invariant this shares with the other backends - + the discarder's rollback must never destroy the keeper's rows - is asserted for all three in + `test_async_connection_is_not_shared_between_concurrent_coroutines`. What is Postgres-only + is the stronger claim that both transactions genuinely ran at once. + + Note that a `contextvars.ContextVar` holding the *pool* would solve nothing: the boundary + has to be a transaction per task, not a per-task reference to a shared one. It also has to + be keyed to the task that acquired it - a `ContextVar` set in a parent is copied into every + task it later spawns, so an unkeyed entry would hand both coroutines below the same + connection and quietly reintroduce exactly the bug this test exists to catch. + + The interleave, pinned with events rather than sleeps so the ordering is deterministic: + - `keeper` inserts `keep`, then commits once `discarder` has rolled back + - `discarder` inserts `discard`, then rolls its own insert back + + Per-task transactions leave only `keep`. One shared transaction leaves `discard` behind: it + was committed out from under the coroutine that asked for it to be discarded. + """ + db = dal_psql + + @db.define() + class AsyncThingSharedTransaction(TypedTable): + name: TypedField[str] + + db.commit() + + keeper_inserted = asyncio.Event() + discarder_rolled_back = asyncio.Event() + + async def keeper(): + await AsyncThingSharedTransaction.insert_async(name="keep") + keeper_inserted.set() + await discarder_rolled_back.wait() + await db.commit_async() + + async def discarder(): + await keeper_inserted.wait() + await AsyncThingSharedTransaction.insert_async(name="discard") + await db.rollback_async() + discarder_rolled_back.set() + + try: + await asyncio.gather(keeper(), discarder()) + + rows = await AsyncThingSharedTransaction.collect_async() + assert sorted(row.name for row in rows) == ["keep"] + finally: + # `db_async` does this in its teardown; `dal_psql` is a plain session db, so an async + # pool left open here outlives this test's event loop and hangs the next one. + await db.close_async() + + +@pytest.mark.asyncio +async def test_split_guard_is_per_task_not_per_instance(dal_psql: TypeDAL): + """ + The sync/async split guard has to be keyed to the task whose transaction it describes. + + `TransactionSplitError` and its two flags exist so a statement on one connection can never + silently miss uncommitted work on the other. On Postgres the async side keeps one + connection *per task* (`PostgresAsyncPool`), so "the async connection has uncommitted + writes" is a per-task fact - but `_async_pending` (core.py) is a single bool on the + `TypeDAL` instance, so any task's `commit_async()` clears it for every other task. + + Three coroutines, pinned with events rather than sleeps: + - `holder` inserts and does *not* end its transaction, so its connection stays dirty + - `bystander` does its own unrelated write and commits it, which is what clears the flag + - `sync_reader` then issues a plain synchronous SELECT + + That SELECT runs on pydal's own connection and cannot see `holder`'s row, which is the + exact condition the guard is there to refuse. It must raise. Today it does not: the flag + `SyncTransactionTracker.before_execute` reads was reset by a task that had no business + speaking for `holder`, so the guard fails *open* - and it fails open only under + concurrency, which is where it is the only thing standing between the caller and a wrong + answer. + + Postgres only, on `dal_psql`, for the same reason as + `test_concurrent_coroutines_do_not_share_one_transaction`: two tasks have to hold separate + open transactions at once for the premise to exist at all, and SQLite permits one writer. + + The mirror defect - `holder`'s write refusing an unrelated task's sync read - is the same + root cause and is not asserted here; fixing the flag to be per-task fixes both. + """ + db = dal_psql + + @db.define() + class AsyncThingSplitGuard(TypedTable): + name: TypedField[str] + + db.commit() + + holder_wrote = asyncio.Event() + bystander_committed = asyncio.Event() + sync_read_done = asyncio.Event() + + # collected rather than raised in place: an exception out of `gather()` propagates while + # the other two coroutines are still running, and the assertion belongs after they are all + # settled anyway. + refusal: list[TransactionSplitError] = [] + + async def holder() -> None: + # never committed or rolled back until the very end - this task's connection is the + # one holding the writes the sync reader must be protected from. + await AsyncThingSplitGuard.insert_async(name="uncommitted") + holder_wrote.set() + await asyncio.wait_for(sync_read_done.wait(), timeout=5) + await db.rollback_async() + + async def bystander() -> None: + # ordinary, correct, unrelated work: its own connection, its own transaction, ended + # properly. Nothing here is a misuse; that is the point. + await asyncio.wait_for(holder_wrote.wait(), timeout=5) + await AsyncThingSplitGuard.insert_async(name="bystander") + await db.commit_async() + bystander_committed.set() + + async def sync_reader() -> None: + await asyncio.wait_for(bystander_committed.wait(), timeout=5) + try: + # a plain INSERT takes no lock a SELECT waits on, so this does not block on + # `holder` - it just quietly returns a view of the table that is missing a row. + AsyncThingSplitGuard.collect() + except TransactionSplitError as e: + refusal.append(e) + finally: + sync_read_done.set() + + try: + await asyncio.gather(holder(), bystander(), sync_reader()) + + assert refusal, ( + "the sync SELECT was allowed to run while another task's async transaction held " + "uncommitted writes - `_async_pending` was cleared by `bystander`, which speaks " + "only for its own connection" + ) + finally: + await db.close_async() + + +@pytest.mark.asyncio +async def test_insert_async_honors_on_insert_error_hook(db_async: TypeDAL): + """ + pydal's `adapter.insert()` routes a failing INSERT through `table._on_insert_error` and + returns the hook's value (adapters/base.py). `db.insert_async()` does not, so the + same table diverges between sync and async on a constraint violation - while the sibling + `update_async()` twenty lines up already does honour `_on_update_error` (core.py). + """ + db = db_async + + @db.define() + class AsyncThingInsertError(TypedTable): + name = TypedField(str, unique=True) + + table = AsyncThingInsertError._ensure_table_defined() + table._on_insert_error = lambda _table, _fields, _e: "handled" + + AsyncThingInsertError.insert(name="dup") + db.commit() + + # sync: the hook swallows the integrity error and its return value comes back out + assert table.insert(name="dup") == "handled" + db.rollback() # the failed statement aborted the sync transaction (postgres) + + # async must do the same: + duplicate = table._fields_and_values_for_insert({"name": "dup"}).op_values() + assert await db.insert_async(table, duplicate) == "handled" + + +@pytest.mark.asyncio +async def test_async_pool_manager_opens_once_under_concurrency(db_async: TypeDAL): + """ + Creating the pool is check-then-assign around an `await`, so two coroutines whose first use + overlaps can both pass the check and both open one: a second psycopg pool, or on SQLite a + second aiosqlite connection. Only one can be stored; the other would be dropped without + `close()`, leaking the connection (and, for aiosqlite, its background thread). + + Driven through a manager of its own with a counting `factories` entry - a constructor + argument, so nothing global is swapped out. The stand-in suspends before doing the real + work: both real factories contain awaits, but whether a given one actually yields is a + driver detail (`aiosqlite.connect()` does, `psycopg_pool.open()` currently does not) and + this is a test of the manager, not of which drivers make the race observable today. + """ + dbengine = db_async._adapter.dbengine + real_factory = ASYNC_POOL_FACTORIES[dbengine] + opened = [] + + async def counting_factory(dal: TypeDAL): + await asyncio.sleep(0) # any await inside a factory is enough to open the window + pool = await real_factory(dal) + opened.append(pool) + return pool + + manager = AsyncPoolManager(db_async, factories={dbengine: counting_factory}) + try: + first, second = await asyncio.gather(manager.get(), manager.get()) + + assert first is second, "concurrent first use handed out two different pools" + assert len(opened) == 1, f"opened {len(opened)}, so {len(opened) - 1} was leaked unclosed" + finally: + kept = manager.pool + await manager.close() + # whatever a leak left behind is no longer the manager's to close: + for pool in opened: + if pool is not kept: + with contextlib.suppress(Exception): + await pool.close() + + +@pytest.mark.asyncio +async def test_update_record_async_ignores_common_filters_like_sync(db_async: TypeDAL): + """ + pydal's `RecordUpdater` writes by primary key with `ignore_common_filters=True` + (helpers/classes.py), so a record you already hold can always be written back. + `update_record_async()` rebuilds that update through `QueryBuilder.update_async()` without + the flag, so `adapter._update()` re-applies the table's common filter (base.py via + `use_common_filters`, helpers/methods.py) and a row the filter excludes - a + soft-deleted one, say - silently updates zero rows. + + Also reached by `validate_and_update_async()` and the update branch of + `update_or_insert_async()`, which both route through `update_record_async()`. + """ + db = db_async + + @db.define() + class AsyncThingCommonFilter(TypedTable): + name: TypedField[str] + archived: TypedField[bool] + + table = AsyncThingCommonFilter._ensure_table_defined() + + row_id = int(AsyncThingCommonFilter.insert(name="original", archived=True)) + db.commit() + + # hold the record from before the filter exists, as a soft-delete flow would + record = AsyncThingCommonFilter.where(AsyncThingCommonFilter.id == row_id).first() + + table._common_filter = lambda _query: table.archived == False # noqa: E712 + + try: + # sync twin writes straight through the filter: + record.update_record(name="sync-updated") + db.commit() + + # async twin must too: + await record.update_record_async(name="async-updated") + await db.commit_async() + finally: + table._common_filter = None + + fresh = AsyncThingCommonFilter.where(AsyncThingCommonFilter.id == row_id).first() + assert fresh.name == "async-updated" + + +@pytest.mark.asyncio +async def test_postgres_lastrowid_async_uses_only_the_value_it_was_given(dal_psql: TypeDAL): + """ + `postgres_lastrowid_async()` must decide whether the statement it just ran carried a + RETURNING clause from its `last_insert` argument alone - never by reading + `adapter._last_insert` back. That attribute is a property over + `THREAD_LOCAL._pydal_last_insert_` (pydal adapters/postgres.py), and coroutines + share one thread, so for the async path it is effectively a global: any other insert + running between `_insert()` and here overwrites it. + + Proven by executing a `DEFAULT VALUES` insert - no fields, therefore no RETURNING + (postgres.py) - while the thread-local says the opposite. Reading the attribute + would take the `fetchone()` branch and raise on a statement that produced no rows. + + Takes the Postgres fixture directly instead of the parametrized `db_async`: SQLite has no + equivalent flag - `sqlite_lastrowid_async` ignores `last_insert` entirely and returns + `cursor.lastrowid` - so there would be nothing for a SQLite run to assert. + """ + async with _postgres_db(dal_psql) as db: + + @db.define() + class AsyncThingLastInsert(TypedTable): + name = TypedField(str, notnull=False) + + table = AsyncThingLastInsert._ensure_table_defined() + adapter = db._adapter + + sql = adapter._insert(table, []) + captured = adapter._last_insert # what *this* statement produced: None + assert captured is None + + # stand-in for a concurrent insert_async() landing between the build and the read: + adapter._last_insert = (table._id, 1) + + pool = await db._get_async_pool() + async with pool.connection() as conn, conn.cursor() as cur: + await cur.execute(sql) + row_id = await postgres_lastrowid_async(adapter, table, cur, captured) + + assert isinstance(row_id, int) + assert row_id > 0 + + +@pytest.mark.asyncio +async def test_async_connection_is_not_shared_between_concurrent_coroutines(db_async: TypeDAL): + """ + Two coroutines writing at the same time must never end up deciding each other's outcome. + + This test used to assert the opposite contract: that `connection()` commits on clean exit + and rolls back on exception, so a failing writer's row disappears and a clean writer's row + survives *because of how the block exited*. That per-call commit is the defect + `test_insert_async_can_be_rolled_back` removes - it put every `_async` write outside + anything the caller could undo - so the two assertions cannot both hold. The isolation + intent is kept here; the auto-commit mechanism it used to rely on is not. + + What replaces it: each writer ends its own transaction explicitly, the way pydal expects. + The outcome asserted is the same one the old test wanted - `keep` survives, `discard` does + not - but it now depends on the transactions being *separate*, not on the context manager + guessing. + + Do NOT rewrite the overlap with an `asyncio.Barrier`. It deadlocks on `sqlite:memory`, and + not because of a bug: that backend refuses a second concurrent transaction outright + (`ConcurrentTransactionError`), so demanding both coroutines be inside at once demands the + thing the design exists to prevent. Each writer instead signals that it is inside and waits + a bounded time for the other, which forces an overlap where one is possible and simply + times out where it is not. + + Per backend, all three of which are safe and none of which lose `keep`: + + - Postgres: a connection per task, genuinely concurrent, both transactions independent. + - file-backed SQLite: a connection per task, but SQLite allows one writer at a time, so + the second waits out `busy_timeout` and then reports `database is locked`. + - `sqlite:memory`: one connection, so the second writer is refused immediately with + `ConcurrentTransactionError`. + + The second writer failing is therefore an accepted outcome on SQLite, and the assertion is + about what the database is left holding rather than about who got to run. + """ + db = db_async + + @db.define() + class AsyncThingIsolation(TypedTable): + name: TypedField[str] + + db.commit() + + keeper_inside = asyncio.Event() + failer_inside = asyncio.Event() + + async def wait_briefly(event: asyncio.Event) -> None: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(event.wait(), timeout=0.25) + + async def committing_writer() -> None: + await AsyncThingIsolation.insert_async(name="keep") + keeper_inside.set() + await wait_briefly(failer_inside) + await db.commit_async() + + async def failing_writer() -> None: + # both are accepted: the write goes through and this rolls it back, or SQLite refuses + # it outright. Either way `discard` must not be in the database at the end. + with contextlib.suppress(TransactionBoundaryError, sqlite3.OperationalError): + await AsyncThingIsolation.insert_async(name="discard") + failer_inside.set() + await wait_briefly(keeper_inside) + await db.rollback_async() + + failer_inside.set() + + await asyncio.gather(committing_writer(), failing_writer()) + + rows = await AsyncThingIsolation.collect_async() + assert sorted(row.name for row in rows) == ["keep"] + +@pytest.mark.asyncio +async def test_rollback_async_is_usable_on_every_backend(db_async: TypeDAL): + """ + `rollback_async()` is a no-op for Postgres (psycopg_pool already rolled back on context + exit) and real work for SQLite, but it must be callable and leave the connection usable + on both - that is the whole point of putting it on `AsyncConnectionPool`. + """ + db = db_async + + @db.define() + class AsyncThingRollback(TypedTable): + qty: TypedField[int] + + await AsyncThingRollback.insert_async(qty=1) + await db.commit_async() + + await db.rollback_async() + + # every `_async` call is its own committed transaction, so the row survives and the + # connection still works afterwards: + assert await AsyncThingRollback.count_async() == 1 + + +@pytest.mark.asyncio +async def test_delete_async_cascades_to_referencing_rows(db_async: TypeDAL): + """ + `sqlite_delete_async` re-implements `SQLite.delete()`'s cascade (adapters/sqlite.py): + select ids, delete, then recurse per FK with `ondelete=CASCADE`. Postgres leaves that to + the database. Either way the children must be gone. + """ + db = db_async + + @db.define() + class AsyncCascadeParent(TypedTable): + name: TypedField[str] + + @db.define() + class AsyncCascadeChild(TypedTable): + parent: AsyncCascadeParent + + parent_id = int(AsyncCascadeParent.insert(name="parent")) + AsyncCascadeChild.insert(parent=parent_id) + AsyncCascadeChild.insert(parent=parent_id) + db.commit() + + assert AsyncCascadeChild.count() == 2 + + await AsyncCascadeParent.where(AsyncCascadeParent.id == parent_id).delete_async() + await db.commit_async() + + assert await AsyncCascadeParent.count_async() == 0 + assert await AsyncCascadeChild.count_async() == 0 + + +@pytest.mark.asyncio +async def test_update_async_honors_on_update_error_hook(db_async: TypeDAL): + """ + Twin of `test_insert_async_honors_on_insert_error_hook`: `update_async` routes a failing + UPDATE through `table._on_update_error`, mirroring `adapter.update()` (base.py). + """ + db = db_async + + @db.define() + class AsyncThingUpdateError(TypedTable): + name = TypedField(str, unique=True) + + table = AsyncThingUpdateError._ensure_table_defined() + table._on_update_error = lambda _table, _query, _fields, _e: -1 + + first = int(AsyncThingUpdateError.insert(name="a")) + AsyncThingUpdateError.insert(name="b") + db.commit() + + # renaming 'a' to 'b' violates the unique constraint: + row = table._fields_and_values_for_update({"name": "b"}) + result = await db.update_async(table, table.id == first, row.op_values()) + + assert result == -1 + + +@pytest.mark.asyncio +async def test_async_pool_manager_rejects_unsupported_backend(db_async: TypeDAL): + """ + A dbengine with no registered factory must fail loudly and name what *is* supported, rather + than KeyError-ing out. Expressed by handing the manager a registry that does not cover this + backend - again a constructor argument, not a patched global or a faked adapter. + """ + manager = AsyncPoolManager(db_async, factories={"nosuchengine": open_sqlite_async_connection}) + + with pytest.raises(NotImplementedError, match="only implemented for nosuchengine"): + await manager.get() + + assert manager.pool is None + + +@pytest.mark.asyncio +async def test_insert_async_runs_pydal_insert_hooks(db_async: TypeDAL): + """ + `TypedTable.insert_async()` keeps pydal's `Table.insert()` hook dance (objects.py): + a truthy `_before_insert` aborts the insert, and `_after_insert` sees the new id. + """ + db = db_async + + @db.define() + class AsyncThingInsertHooks(TypedTable): + qty: TypedField[int] + + table = AsyncThingInsertHooks._ensure_table_defined() + seen: list[t.Any] = [] + + table._after_insert.append(lambda _row, result: seen.append(result)) + await AsyncThingInsertHooks.insert_async(qty=1) + await db.commit_async() + assert len(seen) == 1 + assert await AsyncThingInsertHooks.count_async() == 1 + + # a truthy _before_insert aborts, so nothing is written and no id comes back: + table._before_insert.append(lambda _row: True) + await AsyncThingInsertHooks.insert_async(qty=2) + await db.commit_async() + assert await AsyncThingInsertHooks.count_async() == 1 + assert len(seen) == 1 + + +@pytest.mark.asyncio +async def test_delete_async_runs_pydal_delete_hooks(db_async: TypeDAL): + """ + `QueryBuilder.delete_async()` replicates `Set.delete()`'s hooks (objects.py), + since pydal has no async version to delegate to: a truthy `_before_delete` aborts and + returns no ids, `_after_delete` runs on success, and a query matching nothing returns []. + """ + db = db_async + + @db.define() + class AsyncThingDeleteHooks(TypedTable): + qty: TypedField[int] + + table = AsyncThingDeleteHooks._ensure_table_defined() + AsyncThingDeleteHooks.insert(qty=1) + db.commit() + + # matches nothing -> no ids, and the after hooks must not fire + assert await AsyncThingDeleteHooks.where(AsyncThingDeleteHooks.qty > 99).delete_async() == [] + + # aborted by a truthy _before_delete + aborter = table._before_delete.append(lambda _set: True) or table._before_delete[-1] + assert await AsyncThingDeleteHooks.where(AsyncThingDeleteHooks.qty > 0).delete_async() == [] + assert await AsyncThingDeleteHooks.count_async() == 1 + table._before_delete.remove(aborter) + + # and the success path runs _after_delete + after: list[t.Any] = [] + table._after_delete.append(lambda pydal_set: after.append(pydal_set)) + assert len(await AsyncThingDeleteHooks.where(AsyncThingDeleteHooks.qty > 0).delete_async()) == 1 + assert len(after) == 1 + + +@pytest.mark.asyncio +async def test_update_async_runs_pydal_update_hooks(db_async: TypeDAL): + """ + Same as the delete twin, for `QueryBuilder.update_async()`: no fields is an error, a truthy + `_before_update` aborts, `_after_update` runs on success, and a no-match query returns []. + """ + db = db_async + + @db.define() + class AsyncThingUpdateHooks(TypedTable): + qty: TypedField[int] + + table = AsyncThingUpdateHooks._ensure_table_defined() + AsyncThingUpdateHooks.insert(qty=1) + db.commit() + + with pytest.raises(ValueError, match="No fields to update"): + await AsyncThingUpdateHooks.where(AsyncThingUpdateHooks.qty > 0).update_async() + + # matches nothing -> no ids + assert await AsyncThingUpdateHooks.where(AsyncThingUpdateHooks.qty > 99).update_async(qty=5) == [] + + aborter = table._before_update.append(lambda _set, _row: True) or table._before_update[-1] + assert await AsyncThingUpdateHooks.where(AsyncThingUpdateHooks.qty > 0).update_async(qty=7) == [] + assert await AsyncThingUpdateHooks.count_async() == 1 + table._before_update.remove(aborter) + + after: list[t.Any] = [] + table._after_update.append(lambda pydal_set, _row: after.append(pydal_set)) + assert len(await AsyncThingUpdateHooks.where(AsyncThingUpdateHooks.qty > 0).update_async(qty=7)) == 1 + assert len(after) == 1 + + +@pytest.mark.asyncio +async def test_table_level_count_and_update_or_insert_with_query(db_async: TypeDAL): + """ + Two thin shortcuts the parity tests reach only through a QueryBuilder: `Table.count_async()` + without a `.where(...)`, and `update_or_insert_async()` given a real Query rather than the + DEFAULT/dict forms (`_lookup_query`'s pass-through branch). + """ + db = db_async + + @db.define() + class AsyncThingShortcuts(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + assert await AsyncThingShortcuts.count_async() == 0 + + created = await AsyncThingShortcuts.update_or_insert_async( + AsyncThingShortcuts.name == "widget", + name="widget", + qty=1, + ) + await db.commit_async() + assert created.qty == 1 + assert await AsyncThingShortcuts.count_async() == 1 + + updated = await AsyncThingShortcuts.update_or_insert_async( + AsyncThingShortcuts.name == "widget", + name="widget", + qty=2, + ) + await db.commit_async() + assert updated.qty == 2 + assert await AsyncThingShortcuts.count_async() == 1 + + +@pytest.mark.asyncio +async def test_insert_and_update_async_reraise_without_error_hook(db_async: TypeDAL): + """ + The other half of the `_on_insert_error`/`_on_update_error` branches: with no hook + registered the driver exception must propagate, exactly as pydal's adapter does. + """ + db = db_async + + @db.define() + class AsyncThingNoHook(TypedTable): + name = TypedField(str, unique=True) + + table = AsyncThingNoHook._ensure_table_defined() + first = int(AsyncThingNoHook.insert(name="a")) + AsyncThingNoHook.insert(name="b") + db.commit() + + with pytest.raises(Exception, match=r"(?i)unique"): + await db.insert_async(table, table._fields_and_values_for_insert({"name": "a"}).op_values()) + + # The failed statement aborted the async transaction, so Postgres answers everything after + # it with InFailedSqlTransaction until that transaction ends. The sync twin of this test + # already calls `db.rollback()` for exactly this reason. It only needs saying here now that + # `_async` calls no longer self-commit: before, each one was its own transaction and an + # error could not reach the next. + await db.rollback_async() + + with pytest.raises(Exception, match=r"(?i)unique"): + row = table._fields_and_values_for_update({"name": "b"}) + await db.update_async(table, table.id == first, row.op_values()) + + +@pytest.mark.asyncio +async def test_insert_async_with_custom_primarykey(db_async: TypeDAL): + """ + Tables with a `_primarykey` instead of pydal's standard `_id` report the new row as a + `{name: value}` dict rather than a `Reference` (adapters/base.py). + """ + db = db_async + + table = db.define_table( + "async_pk_thing", + pydal.objects.Field("code", "string"), + pydal.objects.Field("val", "string"), + primarykey=["code"], + ) + db.commit() + + supplied = await db.insert_async(table, [(table.code, "abc"), (table.val, "x")]) + assert supplied == {"code": "abc"} + + # the sibling branch - a keyed table whose pk is *generated* - is unreachable on both + # backends: pydal makes `_primarykey` columns NOT NULL, so an insert that omits the pk + # fails in the database before it could ever be filled in from lastrowid. + + +@pytest.mark.asyncio +async def test_executesql_async_placeholders_and_dict_shapes(db_async: TypeDAL): + """ + `executesql_async` mirrors pydal's `executesql` surface: bound placeholders, `as_dict` / + `as_ordered_dict`, `colnames` overrides, and the duplicate-column guard. + """ + db = db_async + + @db.define() + class AsyncThingSql(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingSql.insert(name="widget", qty=1) + AsyncThingSql.insert(name="gadget", qty=2) + db.commit() + + tablename = str(AsyncThingSql) + placeholder = "%s" if db._adapter.dbengine == "postgres" else "?" + + bound = await db.executesql_async(f"SELECT qty FROM {tablename} WHERE qty > {placeholder}", (1,)) + assert [row[0] for row in bound] == [2] + + as_dicts = await db.executesql_async(f"SELECT name, qty FROM {tablename} ORDER BY qty", as_dict=True) + assert as_dicts == [{"name": "widget", "qty": 1}, {"name": "gadget", "qty": 2}] + + ordered = await db.executesql_async(f"SELECT name, qty FROM {tablename} ORDER BY qty", as_ordered_dict=True) + assert type(ordered[0]) is collections.OrderedDict + assert list(ordered[0]) == ["name", "qty"] + + renamed = await db.executesql_async( + f"SELECT name FROM {tablename} ORDER BY qty", + as_dict=True, + colnames=["label"], + ) + assert renamed[0] == {"label": "widget"} + + with pytest.raises(RuntimeError, match="duplicate column names"): + await db.executesql_async(f"SELECT qty, qty FROM {tablename}", as_dict=True) + + +@pytest.mark.asyncio +async def test_executesql_async_with_fields_and_colnames(db_async: TypeDAL): + """ + Passing `fields` (or `colnames`) routes the raw rows back through `adapter.parse()`, so + values come out typed rather than as driver primitives. + """ + db = db_async + + @db.define() + class AsyncThingParse(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingParse.insert(name="widget", qty=1) + db.commit() + + table = AsyncThingParse._ensure_table_defined() + tablename = str(AsyncThingParse) + + # a whole Table as `fields` expands to its columns... + parsed = await db.executesql_async( + f"SELECT {tablename}.id, {tablename}.name, {tablename}.qty FROM {tablename}", + fields=[table], + ) + assert parsed[0].name == "widget" + assert parsed[0].qty == 1 + + # ...and individual Fields are taken as-is + per_field = await db.executesql_async( + f"SELECT {tablename}.name, {tablename}.qty FROM {tablename}", + fields=[table.name, table.qty], + ) + assert per_field[0].qty == 1 + + # `colnames` without fields resolves the table.column names itself + by_colname = await db.executesql_async( + f"SELECT {tablename}.name FROM {tablename}", + fields=[], + colnames=[f"{tablename}.name"], + ) + assert by_colname[0].name == "widget" + + # ...and `fields` left off entirely is the same case: pydal's own `executesql` treats a + # missing `fields` and an empty one alike (base.py), so both have to reach `parse()` with + # the colnames doing the resolving. + omitted_fields = await db.executesql_async( + f"SELECT {tablename}.name FROM {tablename}", + colnames=[f"{tablename}.name"], + ) + assert omitted_fields[0].name == "widget" + + # a colname without a `table.` prefix is passed through unquoted + bare_colname = await db.executesql_async( + f"SELECT {tablename}.name FROM {tablename}", + fields=[table.name], + colnames=["name"], + ) + assert bare_colname[0].name == "widget" + + +@pytest.mark.asyncio +async def test_executesql_async_on_statement_without_result_set(db_async: TypeDAL): + """ + A statement that produces no rows: psycopg raises on `fetchall()` (caught, -> None) while + sqlite just yields an empty list. Both are acceptable; neither may blow up. + """ + db = db_async + + @db.define() + class AsyncThingNoResult(TypedTable): + qty: TypedField[int] + + db.commit() + + result = await db.executesql_async(f"DELETE FROM {AsyncThingNoResult} WHERE qty < 0") + assert result in (None, []) + + +@pytest.mark.asyncio +async def test_async_query_builder_falls_back_for_plain_pydal_tables(db_async: TypeDAL): + """ + `QueryBuilder` also accepts an old-style pydal table. There is no model to instantiate from + the rows, so `collect_async()` degrades to `execute_async()` and `first_async()` hands back + the raw pydal Row - the async twins of the fallbacks `collect()`/`first()` already have. + """ + db = db_async + + table = db.define_table("async_plain_thing", pydal.objects.Field("qty", "integer")) + table.insert(qty=1) + db.commit() + + rows = await QueryBuilder(table).collect_async() + assert len(rows) == 1 + + row = await QueryBuilder(table).first_async() + assert row is not None + assert row.qty == 1 + + +@pytest.mark.asyncio +async def test_classmethod_update_async_returns_none_when_nothing_matches(db_async: TypeDAL): + """`Model.update_async(query, ...)` mirrors the sync `update()`: no matching row -> None.""" + db = db_async + + @db.define() + class AsyncThingClsUpdateMiss(TypedTable): + qty: TypedField[int] + + db.commit() + + assert await AsyncThingClsUpdateMiss.update_async(AsyncThingClsUpdateMiss.id == 404, qty=1) is None + + +class AsyncThingCached(TypedTable): + """ + Defined at module level, unlike every other model here: the cache pickles the rows, and a + class defined inside a test function is not picklable. + """ + + qty: TypedField[int] + + +@pytest.mark.asyncio +async def test_collect_async_serves_cached_rows(): + """ + A cache hit short-circuits `collect_async()` in `_collect_prepare()`, before it ever reaches + the database. Not parametrized over `db_async`: that fixture disables TypeDAL caching. + """ + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL("sqlite:memory", folder=directory) + try: + db.define(AsyncThingCached) + + AsyncThingCached.insert(qty=1) + db.commit() + + fresh = await AsyncThingCached.where(AsyncThingCached.qty > 0).cache().collect_async() + cached = await AsyncThingCached.where(AsyncThingCached.qty > 0).cache().collect_async() + + assert len(fresh) == len(cached) == 1 + assert fresh.metadata["cache"]["status"] == "fresh" + assert cached.metadata["cache"]["status"] == "cached" + finally: + await db.close_async() + db.close() + +@contextlib.contextmanager +def _fail_after(seconds: float, message: str) -> t.Iterator[None]: + """ + Fail instead of hanging when the code under test loops forever. + + `SIGALRM` rather than `asyncio.timeout()`: the loop this guards (see + `test_executesql_async_accepts_a_single_field`) contains no await, so the event loop never + gets control back and an asyncio timeout would never fire. Signals are only delivered on + the main thread, which is where pytest-asyncio runs the loop. + """ + + def raise_timeout(_signum: int, _frame: t.Any) -> None: + raise TimeoutError(message) + + previous = signal.signal(signal.SIGALRM, raise_timeout) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +@pytest.mark.asyncio +async def test_insert_async_does_not_run_a_sync_query(db_async: TypeDAL): + """ + `insert_async()` may not fall back to the sync connection to build its return value. + + It returns `self(result)` with `result` an int-like `Reference`, which `TypedTable.__new__` + feeds to pydal's synchronous `Table.__call__` -> `db(...).select()`. That is a blocking + SELECT on the event loop, on the *other* (sync) connection, for a row this method already + has the id of. `db._timings` is pydal's own record of every statement executed on the sync + adapter (helpers/classes.py, installed by default via `DAL.execution_handlers`), + so it can be checked without patching anything. + """ + db = db_async + + @db.define() + class AsyncThingInsertBlocking(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + db.commit() + + before = len(db._timings) + inserted = await AsyncThingInsertBlocking.insert_async(name="widget", qty=5) + + # the return value must stay usable - the point is how it is built, not that it shrinks + assert int(inserted) > 0 + assert inserted.name == "widget" + assert inserted.qty == 5 + + sync_statements = [command for command, _ in db._timings[before:]] + selects = [command for command in sync_statements if command.lstrip().upper().startswith("SELECT")] + assert not selects, f"insert_async ran {len(selects)} synchronous SELECT(s): {selects}" + + +@pytest.mark.asyncio +async def test_update_or_insert_async_handles_none_and_false_query(db_async: TypeDAL): + """ + `None` and `False` are both members of `T_Query` (types.py) and both are accepted by + the sync `update_or_insert()`: pydal's `Table.__call__` (objects.py) finds no record + for a non-Query, non-digit key, so the call inserts. The async twin routes the same values + through `QueryBuilder.where()` (query_builder.py), which raises `ValueError`. + """ + db = db_async + + @db.define() + class AsyncThingUpsertFalsy(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + db.commit() + + sync_none = AsyncThingUpsertFalsy.update_or_insert(None, name="via-none", qty=1) + sync_false = AsyncThingUpsertFalsy.update_or_insert(False, name="via-false", qty=2) + db.commit() + assert sync_none.name == "via-none" + assert sync_false.name == "via-false" + + async_none = await AsyncThingUpsertFalsy.update_or_insert_async(None, name="via-none-async", qty=3) + async_false = await AsyncThingUpsertFalsy.update_or_insert_async(False, name="via-false-async", qty=4) + await db.commit_async() + + assert async_none.name == "via-none-async" + assert async_false.name == "via-false-async" + assert AsyncThingUpsertFalsy.count() == 4 + + +@pytest.mark.asyncio +async def test_executesql_async_accepts_a_single_field(db_async: TypeDAL): + """ + pydal's `executesql()` explicitly allows `fields` to be one object instead of a list + (base.py: `if not isinstance(fields, list): fields = [fields]`), and TypeDAL's sync + `executesql()` inherits that by delegating to it. `executesql_async()` does + `list(fields)` instead. + + That does not fail with a `TypeError`: a `Field` is an `Expression`, and + `Expression.__getitem__` (objects.py) answers any integer index with + `self[i:i+1]` - a substring expression - and never raises `IndexError`. `list()` therefore + falls back to the legacy sequence protocol and spins forever, allocating expressions. Hence + the alarm below: an `asyncio` timeout cannot break a CPU-bound loop with no await in it. + """ + db = db_async + + @db.define() + class AsyncThingSingleField(TypedTable): + name: TypedField[str] + qty: TypedField[int] + + AsyncThingSingleField.insert(name="widget", qty=1) + db.commit() + + table = AsyncThingSingleField._ensure_table_defined() + tablename = str(AsyncThingSingleField) + query = f"SELECT {tablename}.qty FROM {tablename}" + + sync_rows = db.executesql(query, fields=table.qty) + assert sync_rows[0].qty == 1 + + with _fail_after(5, "executesql_async(fields=) never returned"): + async_rows = await db.executesql_async(query, fields=table.qty) + + assert async_rows[0].qty == 1 + + +@pytest.mark.asyncio +async def test_after_connection_hook_also_applies_to_the_async_connection(): + """ + `TypeDAL(..., after_connection=...)` is handed to pydal, which runs it on every sync + connection it opens (connection.py). The async factories in `async_execution.py` + open a raw driver connection and never do, so connection-scoped setup the user asked for + (custom functions, PRAGMAs, session settings) is missing on the async side. + + A TEMP table is the backend-neutral way to observe that: it lives on the connection that + created it, so it is visible from the sync connection and absent from the async one. + + SQLite-only: it needs to build its own `TypeDAL` to pass `after_connection`, which the + `db_async` fixture's already-connected Postgres instance cannot. + + Note the fix is not simply calling `adapter._after_connection(adapter)` from the factory: + the hook is handed the pydal adapter and drives the *sync* cursor, so replaying it there + would re-run it against the wrong connection. Making this pass means giving the async + connection an adapter-shaped façade to run the hook against. + """ + statements: list[str] = [] + + def after_connection(adapter: t.Any) -> None: + statements.append("ran") + adapter.execute("CREATE TEMPORARY TABLE async_hook_marker (x INTEGER)") + + with tempfile.TemporaryDirectory() as directory: + # a URI no other test uses, because pydal's connection pool is global and keyed by URI + # (connection.py): a `sqlite:memory` connection left there by an earlier test is + # handed back with `run_hooks=False`, so the hook would never run and the test would be + # measuring the pool instead of the hook. + db = TypeDAL( + "sqlite://after_connection_hook.db", + enable_typedal_caching=False, + folder=directory, + after_connection=after_connection, + ) + try: + # this query is what opens pydal's sync connection - it connects lazily, so the + # hook has not run at construction time - and the TEMP table it selects from only + # exists because the hook ran while that connection was being set up. + assert db.executesql("SELECT * FROM async_hook_marker") == [] + assert statements, "pydal did not run the hook on its own connection - test is meaningless" + + # ...and the async connection is a different connection, which never saw the hook + with pytest.raises(sqlite3.OperationalError, match="no such table"): + await db.executesql_async("SELECT * FROM async_hook_marker") + finally: + await db.close_async() + db.close() + +class _StubConnection: + """Enough of a psycopg AsyncConnection for the pool to hand around and close.""" + + def __init__(self) -> None: + self.closed = False + self.rolled_back = False + + async def rollback(self) -> None: + self.rolled_back = True + + async def close(self) -> None: + self.closed = True + + +class _StubPool: + """A psycopg_pool stand-in whose `putconn` refuses, the way a closed pool does.""" + + def __init__(self, conn: _StubConnection) -> None: + self.conn = conn + self.closed = False + + async def getconn(self) -> _StubConnection: + return self.conn + + async def putconn(self, _conn: _StubConnection) -> None: + raise RuntimeError("pool is closed") + + async def close(self) -> None: + self.closed = True + + +class _StubAsyncCursor: + """A cursor that always answers an empty read result.""" + + async def execute(self, _sql: str, _parameters: t.Any = None) -> None: + return None + + async def fetchone(self) -> tuple[int]: + return (0,) + + async def fetchall(self) -> list[t.Any]: + return [] + + +class _FailingStubAsyncCursor(_StubAsyncCursor): + """A read cursor that raises before a row can be fetched.""" + + async def execute(self, _sql: str, _parameters: t.Any = None) -> None: + raise RuntimeError("cursor failed") + + +class _StubAsyncConnection: + """Enough of a psycopg AsyncConnection for `PostgresAsyncPool` to run a read.""" + + def __init__(self, cursor: _StubAsyncCursor | None = None) -> None: + self.closed = False + self.rolled_back = False + self._cursor = cursor or _StubAsyncCursor() + + @contextlib.asynccontextmanager + async def cursor(self) -> t.AsyncIterator[_StubAsyncCursor]: + yield self._cursor + + async def rollback(self) -> None: + self.rolled_back = True + + async def close(self) -> None: + self.closed = True + + +class _ReadOnlyPool: + """A pool that reports how many of its connections are currently checked out.""" + + def __init__(self, conn: _StubAsyncConnection) -> None: + self.conn = conn + self.checked_out = 0 + self.returned = 0 + + async def getconn(self) -> _StubAsyncConnection: + self.checked_out += 1 + return self.conn + + async def putconn(self, _conn: _StubAsyncConnection) -> None: + self.checked_out -= 1 + self.returned += 1 + + async def close(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_postgres_pool_closes_a_connection_it_cannot_return(): + """ + A connection the pool refuses to take back must be closed, not forgotten. + + Stubs make the pool-return failure deterministic. + """ + conn = _StubConnection() + pool = PostgresAsyncPool(_StubPool(conn)) + + async def abandons_its_transaction() -> None: + await pool._acquire() # never committed, never rolled back + + await asyncio.create_task(abandons_its_transaction()) + # the done-callback schedules the return rather than doing it inline, so yield once + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert conn.rolled_back, "an abandoned transaction must be rolled back before disposal" + assert conn.closed, "a connection the pool refused must be closed, or its socket leaks" + assert conn not in pool._checked_out, "a disposed-of connection must not stay tracked" + + +@pytest.mark.asyncio +async def test_postgres_read_only_async_returns_its_connection(): + """ + Read-only async statements must not retain a Postgres connection after they return. + + The stub pool exposes each public read path's checkout and release. + """ + conn = _StubAsyncConnection() + raw_pool = _ReadOnlyPool(conn) + pool = PostgresAsyncPool(raw_pool) + + async def fake_pool_factory(_db: TypeDAL) -> PostgresAsyncPool: + return pool + + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL("sqlite:memory", enable_typedal_caching=False, folder=directory) + + @db.define() + class AsyncThingReadRelease(TypedTable): + name: TypedField[str] + + db.commit() + + db._async_pools = AsyncPoolManager(db, factories={"sqlite": fake_pool_factory}) + try: + assert await AsyncThingReadRelease.count_async() == 0 + assert raw_pool.checked_out == 0, "count_async left its connection checked out" + assert raw_pool.returned == 1, "count_async's read-only connection was never returned" + + collected = await AsyncThingReadRelease.where(AsyncThingReadRelease.id > 0).collect_async() + assert list(collected) == [] + assert raw_pool.checked_out == 0, "collect_async left its connection checked out" + assert raw_pool.returned == 2, "collect_async's read-only connection was never returned" + + selected = await db.select_async(AsyncThingReadRelease.id > 0, AsyncThingReadRelease.id) + assert list(selected) == [] + assert raw_pool.checked_out == 0, "select_async left its connection checked out" + assert raw_pool.returned == 3, "select_async's read-only connection was never returned" + + table_name = AsyncThingReadRelease._table._rname + assert await db.executesql_async(f"SELECT id FROM {table_name}") == [] + assert raw_pool.checked_out == 0, "executesql_async left its connection checked out" + assert raw_pool.returned == 4, "executesql_async's read-only connection was never returned" + finally: + await db.close_async() + db.close() + + +@pytest.mark.asyncio +async def test_postgres_read_only_async_returns_its_connection_when_cursor_fails(): + """ + A read that raises must still return its Postgres connection via the read-only `finally`. + """ + conn = _StubAsyncConnection(_FailingStubAsyncCursor()) + raw_pool = _ReadOnlyPool(conn) + pool = PostgresAsyncPool(raw_pool) + + async def fake_pool_factory(_db: TypeDAL) -> PostgresAsyncPool: + return pool + + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL("sqlite:memory", enable_typedal_caching=False, folder=directory) + + @db.define() + class AsyncThingReadReleaseFailure(TypedTable): + name: TypedField[str] + + db.commit() + + db._async_pools = AsyncPoolManager(db, factories={"sqlite": fake_pool_factory}) + try: + with pytest.raises(RuntimeError, match="cursor failed"): + await AsyncThingReadReleaseFailure.count_async() + + assert raw_pool.checked_out == 0, "a failed read left its connection checked out" + assert raw_pool.returned == 1, "a failed read's connection was never returned" + assert conn.rolled_back, "a failed read's transaction was not rolled back" + finally: + await db.close_async() + db.close() + + +@pytest.mark.asyncio +async def test_settling_up_twice_is_a_no_op(db_async: TypeDAL): + """ + `commit_async()`/`rollback_async()` must be safe when this task holds no connection. + """ + db = db_async + + @db.define() + class AsyncThingSettleTwice(TypedTable): + name: TypedField[str] + + db.commit() + + # nothing done on the async side yet + await db.commit_async() + await db.rollback_async() + + await AsyncThingSettleTwice.insert_async(name="once") + await db.commit_async() + await db.commit_async() # second one has nothing left to settle + await db.rollback_async() # and this must not undo the commit above + + assert [row.name for row in await AsyncThingSettleTwice.collect_async()] == ["once"] + + +@pytest.mark.asyncio +async def test_sqlite_pool_reclaim_yields_to_whoever_claimed_first(): + """ + A scheduled reclaim must do nothing when another path already claimed the connection. + """ + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL(f"sqlite://{Path(directory) / 'reclaim.db'}", enable_typedal_caching=False, folder=directory) + try: + pool = await db._get_async_pool() + conn = await pool._acquire() + + pool._reclaim(conn) # schedules the rollback-and-close + pool._open.discard(conn) # somebody else claims it before that runs + await asyncio.sleep(0) # let the scheduled work find it gone + + assert not pool._open + + # the losing claim leaves this connection open on purpose - that is the branch + # under test - so close it here. aiosqlite runs a non-daemon thread per connection, + # and one left behind outlives the test's event loop and reports + # `RuntimeError: Event loop is closed` from inside some later, unrelated test. + await conn.close() + finally: + await db.close_async() + db.close() + + +@pytest.mark.asyncio +async def test_sqlite_pool_close_closes_open_connections(): + """`close()` must roll back and close connections still checked out.""" + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL(f"sqlite://{Path(directory) / 'close.db'}", enable_typedal_caching=False, folder=directory) + try: + pool = await db._get_async_pool() + await pool._acquire() + await pool.close() + assert not pool._open + finally: + await db.close_async() + db.close() + + +@pytest.mark.asyncio +async def test_sqlite_pool_reclaim_closes_an_abandoned_connection(): + """`_reclaim`'s scheduled coroutine must roll back and close an abandoned connection.""" + with tempfile.TemporaryDirectory() as directory: + db = TypeDAL(f"sqlite://{Path(directory) / 'reclaim.db'}", enable_typedal_caching=False, folder=directory) + try: + pool = await db._get_async_pool() + conn = await pool._acquire() + pool._reclaim(conn) + await asyncio.sleep(0.1) + assert not pool._open + finally: + await db.close_async() + db.close() + + +@pytest.mark.parametrize( + ("foreign_settle", "keeper_settle", "expected_rows", "message"), + [ + ( + "rollback_async", + "commit_async", + ["keep"], + "a task that holds no transaction rolled back the one another task was still writing to", + ), + ( + "commit_async", + "rollback_async", + [], + "a task that holds no transaction committed the one another task was still writing to", + ), + ], +) +@pytest.mark.asyncio +async def test_sqlite_memory_non_owner_settlement_is_a_no_op( + db_sqlite_memory: TypeDAL, + foreign_settle: str, + keeper_settle: str, + expected_rows: list[str], + message: str, +): + """ + A task that owns no transaction must not commit or roll back the one another task holds. + + Before the ownership guard, `SqliteAsyncConnection.commit()`/`rollback()` acted on the + single shared connection unconditionally; now `_end_transaction()` no-ops for a non-owner. + The keeper writes and then settles its own way after the outsider has had its turn. + """ + db = db_sqlite_memory + + @db.define() + class AsyncThingForeignSettle(TypedTable): + name: TypedField[str] + + db.commit() + + keeper_wrote = asyncio.Event() + outsider_settled = asyncio.Event() + + async def keeper() -> None: + await AsyncThingForeignSettle.insert_async(name="keep") + keeper_wrote.set() + await asyncio.wait_for(outsider_settled.wait(), timeout=5) + await getattr(db, keeper_settle)() + + async def outsider() -> None: + await asyncio.wait_for(keeper_wrote.wait(), timeout=5) + await getattr(db, foreign_settle)() + outsider_settled.set() + + await asyncio.gather(keeper(), outsider()) + + assert [row.name for row in await AsyncThingForeignSettle.collect_async()] == expected_rows, message + + +@pytest.mark.asyncio +async def test_sqlite_memory_does_not_inherit_an_abandoned_transaction(db_sqlite_memory: TypeDAL): + """ + A `sqlite:memory` transaction whose task ended without settling it must not be handed to + the next task. + + `SqliteAsyncConnection` reclaims through `_settle_abandoned_owner()` on the next + `connection()` and through the `_reclaim` done-callback armed by + `_take_ownership_if_in_transaction()`; `settle_abandoned_sync()` is the sync-side path. + Asserted as the outcome rather than by poking at `_owner`. + """ + db = db_sqlite_memory + + @db.define() + class AsyncThingAbandoned(TypedTable): + name: TypedField[str] + + db.commit() + + await _abandon_async_transaction(AsyncThingAbandoned) + + await AsyncThingAbandoned.insert_async(name="mine") + await db.commit_async() + + assert [row.name for row in await AsyncThingAbandoned.collect_async()] == ["mine"], ( + "the next task inherited the abandoned transaction and its commit made another task's " + "uncommitted row durable" + ) + + +@pytest.mark.asyncio +async def test_sqlite_memory_abandoned_transaction_does_not_lock_out_the_sync_side(db_sqlite_memory: TypeDAL): + """ + The sync connection must not be refused after an abandoned async transaction is reclaimed. + + `SqliteAsyncConnection` reclaims abandoned owners through `settle_abandoned_sync()` + (invoked by `TypeDAL._has_pending_async_writes()`) and through + `_settle_abandoned_owner()` on the next `connection()`, so the sync side is free to run. + """ + db = db_sqlite_memory + + @db.define() + class AsyncThingAbandonedLock(TypedTable): + name: TypedField[str] + + db.commit() + + await _abandon_async_transaction(AsyncThingAbandonedLock) + + AsyncThingAbandonedLock.insert(name="sync") + db.commit() + + assert sorted(row.name for row in AsyncThingAbandonedLock.collect()) == ["sync"] + + +@pytest.mark.asyncio +async def test_sqlite_memory_async_connection_settles_a_finished_owner(db_sqlite_memory: TypeDAL): + """ + `connection()` must settle a finished owner even when no sync statement has triggered + `settle_abandoned_sync()` first. + + This is the async backstop behind `SqliteAsyncConnection._settle_abandoned_owner()`. + """ + db = db_sqlite_memory + pool = await db._get_async_pool() + finished = await _set_finished_owner(pool) + + async with pool.connection(): + pass + + assert pool._owner is None + + +@pytest.mark.asyncio +async def test_sqlite_memory_sync_side_stays_refused_while_async_lock_is_held(db_sqlite_memory: TypeDAL): + """ + `settle_abandoned_sync()` must not roll back a finished owner while another coroutine is + mid-statement on the single shared connection. + + The sync side cannot await that other coroutine, so the only correct answer is to keep the + abandoned owner counted and let `TransactionSplitError` refuse the sync statement until the + lock holder finishes. This pins the `_lock.locked()` branch in `SqliteAsyncConnection`. + """ + db = db_sqlite_memory + + @db.define() + class AsyncThingLockHeld(TypedTable): + name: TypedField[str] + + db.commit() + + pool = await db._get_async_pool() + finished = await _set_finished_owner(pool) + + entered = asyncio.Event() + leave = asyncio.Event() + + async def hold_the_connection_lock() -> None: + async with pool._lock: + entered.set() + await leave.wait() + + holder = asyncio.create_task(hold_the_connection_lock()) + await asyncio.wait_for(entered.wait(), timeout=5) + + try: + with pytest.raises(TransactionSplitError): + AsyncThingLockHeld.insert(name="blocked") + + assert pool._owner is finished + finally: + leave.set() + await asyncio.wait_for(holder, timeout=5) + pool._owner = None + + +@pytest.mark.asyncio +async def test_refused_task_is_not_recorded_as_holding_async_writes(db_sqlite_memory: TypeDAL): + """ + A task refused with `ConcurrentTransactionError` opened no transaction and must not be + recorded as holding one. + + `_mark_async_pending()` marks before connection acquisition, so it must un-mark when and + only when the connection was refused before any statement ran. Pins the holder/refused + ordering and the final unrelated sync insert. + """ + db = db_sqlite_memory + + @db.define() + class AsyncThingRefusedMark(TypedTable): + name: TypedField[str] + + db.commit() + + holder_wrote = asyncio.Event() + refusal_happened = asyncio.Event() + holder_settled = asyncio.Event() + + # collected rather than asserted inside the coroutine, so a failure does not tear down the + # gather while the other one is still waiting on an event. + problems: list[str] = [] + + async def holder() -> None: + await AsyncThingRefusedMark.insert_async(name="held") + holder_wrote.set() + await asyncio.wait_for(refusal_happened.wait(), timeout=5) + await db.commit_async() + holder_settled.set() + + async def refused() -> None: + await asyncio.wait_for(holder_wrote.wait(), timeout=5) + with pytest.raises(ConcurrentTransactionError): + await AsyncThingRefusedMark.insert_async(name="refused") + refusal_happened.set() + + await asyncio.wait_for(holder_settled.wait(), timeout=5) + + # this task is still alive, so pruning cannot cover for the stale entry + if db._has_pending_async_writes(): + problems.append("a refused task is recorded as holding uncommitted async writes") + + try: + AsyncThingRefusedMark.insert(name="sync") + db.commit() + except TransactionSplitError: + problems.append("a refused task's stale entry refused an unrelated sync statement") + + await asyncio.gather(holder(), refused()) + + assert not problems, problems diff --git a/tests/test_json.py b/tests/test_json.py index 871719c..93dbcc4 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -2,7 +2,7 @@ import json from src.typedal.serializers.as_json import SerializedJson, encode -from typedal.helpers import utcnow +from src.typedal.helpers import utcnow class CustomClass: diff --git a/tests/test_py4web.py b/tests/test_py4web.py index 069ab30..c05e9b8 100644 --- a/tests/test_py4web.py +++ b/tests/test_py4web.py @@ -7,7 +7,7 @@ from src.typedal import TypedTable from src.typedal.for_py4web import DAL, AuthUser, setup_py4web_tables from src.typedal.serializers import as_json -from typedal.config import TypeDALConfig +from src.typedal.config import TypeDALConfig db = DAL("sqlite:memory") diff --git a/tests/test_query_builder.py b/tests/test_query_builder.py index 6c7b92b..3b63622 100644 --- a/tests/test_query_builder.py +++ b/tests/test_query_builder.py @@ -2,8 +2,8 @@ from pydal.objects import Field, Query from src.typedal import TypeDAL, TypedField, TypedTable, relationship -from typedal import QueryBuilder -from typedal.fields import rname +from src.typedal import QueryBuilder +from src.typedal.fields import rname db = TypeDAL("sqlite:memory")