From 18d35e5606b766715c71bfa1d9a3f71bf26ef1d5 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Sun, 23 Mar 2025 16:33:31 +0100 Subject: [PATCH 1/8] chore(deps): bump sqlalchemy from 1.3.24 to 1.4.7 Trying to adapt to SQLAlchemy breaking changes, with the first 1.4 version permitted by dependencies: zope-sqlalchemy==3.1 depends on one of: sqlalchemy>=1.1,<1.4.0 sqlalchemy>1.4.6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87886b16a..84fe5b3cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ redis==5.2.1 requests==2.32.3 setuptools==77.0.3 Shapely==2.0.7 -SQLAlchemy==1.3.24 +SQLAlchemy==1.4.7 transaction==5.0 waitress==3.0.2 zope.sqlalchemy==3.1 From aa5d377e54cac51d94f7ea1e28275629cda7079a Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Sun, 23 Mar 2025 19:05:03 +0100 Subject: [PATCH 2/8] pytest: show SQLAlchemy deprecation warning --- pytest.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index b0e5a945f..3ee4349c7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,7 @@ [pytest] filterwarnings = - ignore::DeprecationWarning \ No newline at end of file + ignore::DeprecationWarning:cornice + ignore::DeprecationWarning:pyramid + ignore::DeprecationWarning:elasticsearch + ignore::DeprecationWarning:pkg_resources + ignore::DeprecationWarning:jwt \ No newline at end of file From 75af3ebe4a0a3efc8bfba8c6e8794089fdff5c02 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Sun, 23 Mar 2025 19:07:32 +0100 Subject: [PATCH 3/8] SQLA: Deprecated since version 1.4: SelectBase.c https://docs.sqlalchemy.org/en/20/core/selectable.html#sqlalchemy.sql.expression.SelectBase.c --- c2corg_api/models/association_views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/c2corg_api/models/association_views.py b/c2corg_api/models/association_views.py index d2fea4ec9..d5f2fe034 100644 --- a/c2corg_api/models/association_views.py +++ b/c2corg_api/models/association_views.py @@ -96,7 +96,7 @@ def _get_select_waypoints_for_routes_aggregated(): type_=postgresql.ARRAY(Integer)).label('waypoint_ids') ]). \ select_from(all_waypoints). \ - group_by(all_waypoints.c.route_id) + group_by(all_waypoints.c.route_id).subquery() class WaypointsForRoutesView(Base): @@ -157,7 +157,7 @@ def _get_select_waypoints_for_outings_aggregated(): type_=postgresql.ARRAY(Integer)).label('waypoint_ids') ]). \ select_from(waypoints_for_outings). \ - group_by(waypoints_for_outings.c.outing_id) + group_by(waypoints_for_outings.c.outing_id).subquery() class WaypointsForOutingsView(Base): @@ -200,7 +200,7 @@ def _get_select_users_for_outings_aggregated(): Association.parent_document_type == user_type, Association.child_document_type == outing_type )). \ - group_by(Association.child_document_id) + group_by(Association.child_document_id).subquery() class UsersForOutingsView(Base): @@ -243,7 +243,7 @@ def _get_select_routes_for_outings_aggregated(): Association.parent_document_type == route_type, Association.child_document_type == outing_type )). \ - group_by(Association.child_document_id) + group_by(Association.child_document_id).subquery() class RoutesForOutingsView(Base): @@ -280,7 +280,7 @@ def _get_select_users_for_routes_aggregated(): type_=postgresql.ARRAY(Integer)).label('user_ids') ]). \ select_from(DocumentTag). \ - group_by(DocumentTag.document_id) + group_by(DocumentTag.document_id).subquery() class UsersForRoutesView(Base): From 1c07801e2eddd59946639df50871a49e2ae04fc4 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Sun, 23 Mar 2025 19:09:50 +0100 Subject: [PATCH 4/8] SAWarning: implicitly coercing SELECT object to scalar subquery; please use the .scalar_subquery() method to produce a scalar subquery. --- c2corg_api/models/feed.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/c2corg_api/models/feed.py b/c2corg_api/models/feed.py index 941d4b50e..61246e298 100644 --- a/c2corg_api/models/feed.py +++ b/c2corg_api/models/feed.py @@ -58,7 +58,8 @@ class FilterArea(Base): User.has_area_filter = column_property( select([func.count(FilterArea.area_id) > 0]). where(FilterArea.user_id == User.id). - correlate_except(FilterArea), + correlate_except(FilterArea). + scalar_subquery(), deferred=True ) User.feed_filter_areas = relationship('Area', secondary=FilterArea.__table__) @@ -98,7 +99,8 @@ class FollowedUser(Base): User.is_following_users = column_property( select([func.count(FollowedUser.followed_user_id) > 0]). where(FollowedUser.follower_user_id == User.id). - correlate_except(FollowedUser), + correlate_except(FollowedUser). + scalar_subquery(), deferred=True ) @@ -474,7 +476,7 @@ def update_langs_of_changes(document_id): DBSession.execute( DocumentChange.__table__.update(). where(DocumentChange.document_id == document_id). - values(langs=langs.select())) + values(langs=langs.select().scalar_subquery())) def get_linked_document(images_in): From 1b0001800cb49a1698a125c1f9916ebf937d5e90 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Sun, 23 Mar 2025 19:10:43 +0100 Subject: [PATCH 5/8] SAWarning: SELECT statement has a cartesian product between FROM element(s) "users.user" and FROM element "users.token". Apply join condition(s) between each element to resolve. --- c2corg_api/security/roles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2corg_api/security/roles.py b/c2corg_api/security/roles.py index 0db6153f3..fcb23db44 100644 --- a/c2corg_api/security/roles.py +++ b/c2corg_api/security/roles.py @@ -50,7 +50,7 @@ def is_valid_token(token): def add_or_retrieve_token(value, expire, userid): token = DBSession.query(Token). \ - filter(Token.value == value, User.id == userid).first() + filter(Token.value == value, Token.userid == userid).first() if not token: token = Token(value=value, expire=expire, userid=userid) DBSession.add(token) From eaa8d1045fb29b7a6d69d5334cd2bbaf3815f034 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Mon, 24 Mar 2025 07:40:29 +0100 Subject: [PATCH 6/8] fix: sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) can't adapt type 'Row' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The error: [SQL: SELECT guidebook.history_metadata.id, guidebook.history_metadata.user_id, guidebook.history_metadata.comment, guidebook.history_metadata.written_at FROM guidebook.history_metadata WHERE guidebook.history_metadata.id IN (%(id_1_1)s, %(id_1_2)s)] [parameters: {'id_1_1': (11,), 'id_1_2': (12,)}] (Background on this error at: http://sqlalche.me/e/14/f405) In this case, it is related to: So this is the psycopg2 driver being given the value (1, ), a one-element tuple, as a parameter that is ultimately bound to a SQL string given the way that psycopg2 binds parameters. psycopg2 seems to just accept tuples and silently flattens them into their lone element, unless you pass more than one element: [...] unfortunately psycopg2 is simply wrong here and your program has been relying on a driver bug — https://github.com/sqlalchemy/sqlalchemy/discussions/9214#discussioncomment-4844748 --- c2corg_api/views/document_delete.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/c2corg_api/views/document_delete.py b/c2corg_api/views/document_delete.py index d061fdb88..bec0b6191 100644 --- a/c2corg_api/views/document_delete.py +++ b/c2corg_api/views/document_delete.py @@ -256,8 +256,8 @@ def _delete_document(self, document_id, document_type, redirecting=False): def _remove_merged_documents(self, document_id, document_type): merged_document_ids = DBSession.query(ArchiveDocument.document_id). \ filter(ArchiveDocument.redirects_to == document_id).all() - for merged_document_id in merged_document_ids: - self._delete_document(merged_document_id, document_type, True) + for row in merged_document_ids: + self._delete_document(row.document_id, document_type, True) @resource(path='/documents/delete/{id}', cors_policy=cors_policy) @@ -365,10 +365,11 @@ def remove_from_cache(document_id): def _remove_versions(document_id): - history_metadata_ids = DBSession. \ - query(DocumentVersion.history_metadata_id). \ - filter(DocumentVersion.document_id == document_id). \ - all() + history_metadata_ids = [ + row.history_metadata_id for row in + DBSession.query(DocumentVersion.history_metadata_id)\ + .filter(DocumentVersion.document_id == document_id).all() + ] DBSession.query(DocumentVersion). \ filter(DocumentVersion.document_id == document_id). \ delete() @@ -391,8 +392,11 @@ def _remove_locale_versions(document_id, lang): subquery('t') # Gets the list of history_metadata_id associated only # to the current locale: - history_metadata_ids = DBSession.query(t.c.history_metadata_id). \ - filter(t.c.lang == lang).filter(t.c.cnt == 1).all() + history_metadata_ids = [ + row.history_metadata_id + for row in DBSession.query(t.c.history_metadata_id)\ + .filter(t.c.lang == lang).filter(t.c.cnt == 1).all() + ] DBSession.query(DocumentVersion). \ filter(DocumentVersion.document_id == document_id). \ From 2ac485eb95e114b75987f956ba32730d8eff033b Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Tue, 25 Mar 2025 08:48:43 +0100 Subject: [PATCH 7/8] SAWarning: Coercing Subquery object into a select() for use in IN(); please pass a select() construct explicitly --- c2corg_api/views/document_delete.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/c2corg_api/views/document_delete.py b/c2corg_api/views/document_delete.py index bec0b6191..a5e4a75dc 100644 --- a/c2corg_api/views/document_delete.py +++ b/c2corg_api/views/document_delete.py @@ -417,7 +417,7 @@ def _remove_archive_locale(archive_clazz_locale, document_id, lang=None): archive_document_locale_ids = query.subquery() DBSession.execute(archive_clazz_locale.__table__.delete().where( getattr(archive_clazz_locale, 'id').in_( - archive_document_locale_ids) + archive_document_locale_ids.select()) )) query = DBSession.query(ArchiveDocumentLocale). \ @@ -435,12 +435,12 @@ def _remove_locale(clazz_locale, document_id, lang=None): document_locale_ids = query.subquery() # Remove links to comments (comments themselves are not removed) DBSession.execute(DocumentTopic.__table__.delete().where( - DocumentTopic.document_locale_id.in_(document_locale_ids) + DocumentTopic.document_locale_id.in_(document_locale_ids.select()) )) if clazz_locale: DBSession.execute(clazz_locale.__table__.delete().where( - getattr(clazz_locale, 'id').in_(document_locale_ids) + getattr(clazz_locale, 'id').in_(document_locale_ids.select()) )) query = DBSession.query(DocumentLocale). \ @@ -467,7 +467,7 @@ def _remove_archive(archive_clazz, document_id): filter(ArchiveDocument.document_id == document_id). \ subquery() DBSession.execute(archive_clazz.__table__.delete().where( - getattr(archive_clazz, 'id').in_(archive_document_ids) + getattr(archive_clazz, 'id').in_(archive_document_ids.select()) )) DBSession.query(ArchiveDocument). \ From 189f6e01a21aa0dd76daa6fc991a66ba3f8b64fa Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Tue, 25 Mar 2025 15:08:17 +0100 Subject: [PATCH 8/8] =?UTF-8?q?Parent=20instance=20=20is=20not=20bound=20to=20a=20Session;=20deferred=20?= =?UTF-8?q?load=20operation=20of=20attribute=20'document=5Fid'=20cannot=20?= =?UTF-8?q?proceed=20(Background=20on=20this=20error=20at:=20http://sqlalc?= =?UTF-8?q?he.me/e/14/bhk3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a workaround as I was unable to fix it in another way (either through `sqlalchemy.orm.undefer` or eager. --- c2corg_api/views/document_info.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/c2corg_api/views/document_info.py b/c2corg_api/views/document_info.py index 7e5509950..c8d629dfd 100644 --- a/c2corg_api/views/document_info.py +++ b/c2corg_api/views/document_info.py @@ -89,12 +89,20 @@ def _load_document_info(self, document_id, lang, clazz): filter(getattr(clazz, 'document_id') == document_id). \ options(joinedload(locales_type_eager). load_only(*locales_load_only)) + document_query = add_load_for_profiles(document_query, clazz) document = document_query.first() if not document: raise HTTPNotFound('document not found') + if document.document_id: + # TODO: find a better way than this workaround which calls + # `document_id` before `set_best_locale` expunge the object + # leading in: sqlalchemy.orm.exc.DetachedInstanceError: + # Parent instance
is not bound to a Session; + # deferred load operation of attribute 'document_id' cannot proceed + pass set_best_locale([document], lang) if document.redirects_to: