chore(firestore): optimize system tests runtime (#17418)

Optimizes Firestore system tests execution time to 13 minutes (down from
~1 hour) under live GCP environments.

**Parallelism:** Added pytest-xdist using all cores (--dist load for
maximum load balancing).
**Isolation:** Appended os.getpid() to test resource IDs to prevent
parallel worker collisions.
**Fixture Re-use:** Promoted query setup fixtures to module scope to
reduce database setups by ~97%.
**Fast Polling:** Reduced hardcoded delays in watch tests from 1.0s to
0.2s with early-exit polling.

Author: ohmayr

.librarian/generator-input/client-post-processing/firestore-integration.yaml

@@ -564,6 +564,9 @@ replacements:
           "freezegun",
       ]
     count: 1
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17429):
+  # Temporary post-processing rule to add pytest-xdist dependency.
+  # Remove this once gapic-generator includes pytest-xdist by default.
   - paths: [
       packages/google-cloud-firestore/noxfile.py
     ]
@@ -574,6 +577,7 @@ replacements:
           "pytest-asyncio",
           "six",
           "pyyaml",
+          "pytest-xdist",
       ]
     count: 1
   - paths: [
@@ -584,6 +588,54 @@ replacements:
     after: |
       "pytest-asyncio==0.21.2",
     count: 2
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17429):
+  # Temporary post-processing rule to inject `-n auto` for Firestore parallel tests.
+  # This rule should be removed once the generator template changes are released
+  # and the generator version is updated in librarian.yaml.
+  - paths: [
+      packages/google-cloud-firestore/noxfile.py
+    ]
+    before: |
+      # Run py.test against the system tests\.
+      if system_test_exists:
+          session\.run\(
+              "py.test",
+              "--quiet",
+              f"--junitxml=system_\{session\.python\}_sponge_log\.xml",
+              system_test_path,
+              \*session\.posargs,
+          \)
+      if system_test_folder_exists:
+          session\.run\(
+              "py.test",
+              "--quiet",
+              f"--junitxml=system_\{session\.python\}_sponge_log\.xml",
+              system_test_folder_path,
+              \*session\.posargs,
+          \)
+    after: |
+      # Run py.test against the system tests.
+      if system_test_exists:
+          session.run(
+              "py.test",
+              "-n",
+              "auto",
+              "--quiet",
+              f"--junitxml=system_{session.python}_sponge_log.xml",
+              system_test_path,
+              *session.posargs,
+          )
+      if system_test_folder_exists:
+          session.run(
+              "py.test",
+              "-n",
+              "auto",
+              "--quiet",
+              f"--junitxml=system_{session.python}_sponge_log.xml",
+              system_test_folder_path,
+              *session.posargs,
+          )
+    count: 1
   - paths: [
       "packages/google-cloud-firestore/docs/conf.py",
     ]

packages/google-cloud-firestore/noxfile.py

@@ -82,6 +82,7 @@
     "pytest-asyncio==0.21.2",
     "six",
     "pyyaml",
+    "pytest-xdist",
 ]
 SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = []
 SYSTEM_TEST_DEPENDENCIES: List[str] = []
@@ -402,6 +403,8 @@ def system(session):
     if system_test_exists:
         session.run(
             "py.test",
+            "-n",
+            "auto",
             "--quiet",
             f"--junitxml=system_{session.python}_sponge_log.xml",
             system_test_path,
@@ -410,6 +413,8 @@ def system(session):
     if system_test_folder_exists:
         session.run(
             "py.test",
+            "-n",
+            "auto",
             "--quiet",
             f"--junitxml=system_{session.python}_sponge_log.xml",
             system_test_folder_path,

packages/google-cloud-firestore/tests/system/test__helpers.py

@@ -16,7 +16,7 @@
 MISSING_DOCUMENT = "No document to update: "
 DOCUMENT_EXISTS = "Document already exists: "
 ENTERPRISE_MODE_ERROR = "only allowed on ENTERPRISE mode"
-UNIQUE_RESOURCE_ID = unique_resource_id("-")
+UNIQUE_RESOURCE_ID = unique_resource_id("-") + "-" + str(os.getpid())
 EMULATOR_CREDS = EmulatorCreds()
 FIRESTORE_EMULATOR = os.environ.get(_FIRESTORE_EMULATOR_HOST) is not None
 FIRESTORE_OTHER_DB = os.environ.get("SYSTEM_TESTS_DATABASE", "system-tests-named-db")

packages/google-cloud-firestore/tests/system/test_system.py

@@ -1268,7 +1268,7 @@ def test_unicode_doc(client, cleanup, database):
     assert snapshot2.reference.id == explicit_doc_id
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def query_docs(client, database):
     collection_id = "qs" + UNIQUE_RESOURCE_ID
     sub_collection = "child" + UNIQUE_RESOURCE_ID
@@ -1297,13 +1297,13 @@ def query_docs(client, database):
         operation()
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def collection(query_docs):
     collection, _, _ = query_docs
     return collection
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def query(collection):
     return collection.where(filter=FieldFilter("a", "==", 1))
 
@@ -2336,7 +2336,11 @@ def test_watch_document(client, cleanup, database):
     doc_ref.set({"first": "Jane", "last": "Doe", "born": 1900})
     cleanup(doc_ref.delete)
 
-    sleep(1)
+    # TODO(https://github.com/googleapis/google-cloud-python/issues/17428):
+    # Investigate why these sleep/polling delays are needed for listener tests.
+    # Having arbitrary delays is fragile and can lead to flakiness.
+    # Explore event-driven synchronization.
+    sleep(0.2)
 
     # Setup listener
     def on_snapshot(docs, changes, read_time):
@@ -2349,12 +2353,12 @@ def on_snapshot(docs, changes, read_time):
     # Alter document
     doc_ref.set({"first": "Ada", "last": "Lovelace", "born": 1815})
 
-    sleep(1)
+    sleep(0.2)
 
-    for _ in range(10):
+    for _ in range(50):
         if on_snapshot.called_count > 0:
             break
-        sleep(1)
+        sleep(0.2)
 
     if on_snapshot.called_count not in (1, 2):
         raise AssertionError(
@@ -2384,15 +2388,19 @@ def on_snapshot(docs, changes, read_time):
 
     collection_ref.on_snapshot(on_snapshot)
 
+    # TODO(https://github.com/googleapis/google-cloud-python/issues/17428):
+    # Investigate why these sleep/polling delays are needed for listener tests.
+    # Having arbitrary delays is fragile and can lead to flakiness.
+    # Explore event-driven synchronization.
     # delay here so initial on_snapshot occurs and isn't combined with set
-    sleep(1)
+    sleep(0.2)
 
     doc_ref.set({"first": "Ada", "last": "Lovelace", "born": 1815})
 
-    for _ in range(10):
+    for _ in range(50):
         if on_snapshot.born == 1815:
             break
-        sleep(1)
+        sleep(0.2)
 
     if on_snapshot.born != 1815:
         raise AssertionError(
@@ -2411,7 +2419,11 @@ def test_watch_query(client, cleanup, database):
     doc_ref.set({"first": "Jane", "last": "Doe", "born": 1900})
     cleanup(doc_ref.delete)
 
-    sleep(1)
+    # TODO(https://github.com/googleapis/google-cloud-python/issues/17428):
+    # Investigate why these sleep/polling delays are needed for listener tests.
+    # Having arbitrary delays is fragile and can lead to flakiness.
+    # Explore event-driven synchronization.
+    sleep(0.2)
 
     # Setup listener
     def on_snapshot(docs, changes, read_time):
@@ -2429,10 +2441,10 @@ def on_snapshot(docs, changes, read_time):
     # Alter document
     doc_ref.set({"first": "Ada", "last": "Lovelace", "born": 1815})
 
-    for _ in range(10):
+    for _ in range(50):
         if on_snapshot.called_count == 1:
             return
-        sleep(1)
+        sleep(0.2)
 
     if on_snapshot.called_count != 1:
         raise AssertionError(
@@ -2806,7 +2818,11 @@ def on_snapshot(docs, changes, read_time):
     on_snapshot.failed = None
     query_ref.on_snapshot(on_snapshot)
 
-    sleep(1)
+    # TODO(https://github.com/googleapis/google-cloud-python/issues/17428):
+    # Investigate why these sleep/polling delays are needed for listener tests.
+    # Having arbitrary delays is fragile and can lead to flakiness.
+    # Explore event-driven synchronization.
+    sleep(0.2)
 
     doc_ref1.set({"first": "Ada", "last": "Lovelace", "born": 1815})
     cleanup(doc_ref1.delete)
@@ -2823,10 +2839,10 @@ def on_snapshot(docs, changes, read_time):
     doc_ref5.set({"first": "Ada", "last": "lovelace", "born": 1815})
     cleanup(doc_ref5.delete)
 
-    for _ in range(10):
+    for _ in range(50):
         if on_snapshot.last_doc_count == 5:
             break
-        sleep(1)
+        sleep(0.2)
 
     if on_snapshot.failed:
         raise on_snapshot.failed

packages/google-cloud-firestore/tests/system/test_system_async.py

@@ -1243,7 +1243,7 @@ async def test_list_collections_with_read_time(client, cleanup, database):
     }
 
 
-@pytest_asyncio.fixture
+@pytest_asyncio.fixture(scope="module")
 async def query_docs(client):
     collection_id = "qs" + UNIQUE_RESOURCE_ID
     sub_collection = "child" + UNIQUE_RESOURCE_ID
@@ -1272,13 +1272,13 @@ async def query_docs(client):
         await operation()
 
 
-@pytest_asyncio.fixture
+@pytest_asyncio.fixture(scope="module")
 async def collection(query_docs):
     collection, _, _ = query_docs
     yield collection
 
 
-@pytest_asyncio.fixture
+@pytest_asyncio.fixture(scope="module")
 async def async_query(collection):
     return collection.where(filter=FieldFilter("a", "==", 1))