fix: unbreak benchmark pipeline (ty errors + Python selection)

.github/workflows/benchmark.yml

@@ -26,87 +26,33 @@ jobs:
 
     - name: Set up Python
       run: |
-        # Automatically detect compatible Python version by testing dependency installation
-        # Try default latest Python first, then downgrade up to 2 times if needed
-
-        echo "Attempting to find compatible Python version..."
-
-        # Try installing default/latest Python
-        uv python install
-
-        # Get the initial version that was installed
-        test_version=$(uv python list | grep -E 'cpython-[0-9]+\.[0-9]+' | grep -v 'pypy' | head -n1 | awk '{print $1}' | sed 's/cpython-//' | cut -d'-' -f1)
-
-        for attempt in 1 2 3; do
-          echo ""
-          echo "Attempt $attempt:"
-          echo "Testing Python $test_version..."
-
-          # Create a temporary virtual environment with the specific Python version
-          # Use attempt number to avoid path collisions
-          test_venv="/tmp/test-venv-attempt-$attempt-$$"
-          uv venv --python "$test_version" "$test_venv" 2>&1
-          source "$test_venv/bin/activate"
-
-          # Try to compile requirements (capture output separately for error reporting)
-          compile_output=$(mktemp)
-          success=false
-          if uv pip compile requirements.in > /tmp/requirements-$attempt.txt 2>"$compile_output"; then
-            # Compile succeeded, now try to sync
-            if uv pip sync /tmp/requirements-$attempt.txt 2>&1; then
-              echo "✓ Python $test_version is compatible with all frameworks"
-              echo "PYTHON_VERSION=$test_version" >> $GITHUB_ENV
-              success=true
-            else
-              echo "✗ Python $test_version: pip sync failed"
-            fi
-          else
-            echo "✗ Python $test_version: dependency compilation failed"
-          fi
-
-          # Show the error for debugging if failed
-          if [ "$success" = false ]; then
-            echo "Error output:"
-            cat "$compile_output" 2>/dev/null || true
-          fi
-
-          # Clean up test venv
-          deactivate 2>/dev/null || true
-          rm -rf "$test_venv"
-          rm -f "$compile_output"
-
-          # If successful, we're done
-          if [ "$success" = true ]; then
-            break
-          fi
-
-          # If not the last attempt, try downgrading
-          if [ $attempt -lt 3 ]; then
-            # Parse version and downgrade
-            major=$(echo "$test_version" | cut -d. -f1)
-            minor=$(echo "$test_version" | cut -d. -f2)
-
-            # Downgrade minor version
-            new_minor=$((minor - 1))
-
-            if [ $new_minor -ge 10 ]; then
-              test_version="$major.$new_minor"
-              echo "Downgrading to Python $test_version..."
-
-              # Install downgraded version
-              uv python install "$test_version"
-            else
-              echo "Cannot downgrade further (reached Python 3.10)"
-              exit 1
-            fi
-          else
-            echo "Error: No compatible Python version found after 3 attempts"
-            exit 1
-          fi
-        done
-
-        echo ""
-        echo "Using Python $PYTHON_VERSION"
+        # fireducks is the framework that lags newest CPython, so its published
+        # wheels decide the version: take the highest cp3XX Linux wheel tag of
+        # the latest fireducks release. Auto-adopts 3.14+ once fireducks ships it.
+        minor=$(curl -sSf --max-time 30 --retry 2 https://pypi.org/pypi/fireducks/json \
+          | jq -r '.urls[].filename
+                   | select(test("manylinux"))
+                   | capture("-cp3(?<m>[0-9]+)-").m' \
+          | sort -n | tail -1)
+
+        if [ -z "$minor" ]; then
+          echo "Error: found no manylinux wheel for the latest fireducks release"
+          exit 1
+        fi
+
+        version="3.$minor"
+        echo "Latest fireducks supports up to Python $version"
+
+        # Confirm every other dependency resolves there too, so an incompatible
+        # pin fails here with a clear message rather than midway through run.sh.
+        uv python install "$version"
+        if ! uv pip compile --python-version "$version" -q requirements.in > /dev/null; then
+          echo "Error: dependencies do not resolve on Python $version"
+          exit 1
+        fi
+
+        echo "PYTHON_VERSION=$version" >> "$GITHUB_ENV"
+        echo "Using Python $version"
 
     - name: Install system dependencies
       run: |

02_benchmark.py

@@ -102,24 +102,28 @@ def groupby_aggregation_operation():
         time_operation(
             "complex_multi_join",
             df_lib,
-            lambda: orders.merge(customers, on="customer_id")
-            .merge(order_items, on="order_id")
-            .merge(products, on="product_id")
-            .sort_values(["order_id", "order_item_id"])
-            .reset_index(drop=True),
+            lambda: (
+                orders.merge(customers, on="customer_id")
+                .merge(order_items, on="order_id")
+                .merge(products, on="product_id")
+                .sort_values(["order_id", "order_item_id"])
+                .reset_index(drop=True)
+            ),
         )
     )
 
     results.append(
         time_operation(
             "four_table_join",
             df_lib,
-            lambda: customers.merge(orders, on="customer_id")
-            .merge(order_items, on="order_id")
-            .merge(products, on="product_id")
-            .merge(reviews, on=["customer_id", "product_id"])
-            .sort_values("customer_id")
-            .reset_index(drop=True),
+            lambda: (
+                customers.merge(orders, on="customer_id")
+                .merge(order_items, on="order_id")
+                .merge(products, on="product_id")
+                .merge(reviews, on=["customer_id", "product_id"])
+                .sort_values("customer_id")
+                .reset_index(drop=True)
+            ),
         )
     )
 
@@ -272,10 +276,12 @@ def rolling_operations_func():
         time_operation(
             "conditional_join",
             df_lib,
-            lambda: customers.merge(orders, on="customer_id")
-            .query("age > 25 and total_amount > 100")
-            .sort_values("customer_id")
-            .reset_index(drop=True),
+            lambda: (
+                customers.merge(orders, on="customer_id")
+                .query("age > 25 and total_amount > 100")
+                .sort_values("customer_id")
+                .reset_index(drop=True)
+            ),
         )
     )
 
@@ -319,9 +325,13 @@ def rolling_operations_func():
         time_operation(
             "time_series_resample",
             df_lib,
-            lambda: time_series.set_index("date")
-            .resample("ME")
-            .agg({"sales": "sum", "marketing_spend": "sum", "website_visits": "mean"}),
+            lambda: (
+                time_series.set_index("date")
+                .resample("ME")
+                .agg(
+                    {"sales": "sum", "marketing_spend": "sum", "website_visits": "mean"}
+                )
+            ),
         )
     )
 

05_comparison.py

@@ -410,7 +410,7 @@ def generate_summary_statistics_markdown(summary_stats: pd.DataFrame) -> str:
     markdown += "## Framework Performance Overview\n\n"
 
     # Convert to markdown table
-    markdown += summary_stats.to_markdown() + "\n\n"
+    markdown += f"{summary_stats.to_markdown()}\n\n"
 
     # Add some interpretation
     markdown += "## Key Metrics Explanation\n\n"
@@ -429,11 +429,11 @@ def generate_summary_statistics_markdown(summary_stats: pd.DataFrame) -> str:
     # Reset index to make framework and cache_status regular columns for sorting
     sorted_stats = summary_stats.reset_index().sort_values("mean")
 
-    for i, row in enumerate(sorted_stats.itertuples(), 1):
-        framework = row.framework
-        cache_status = row.cache_status
-        avg_time = row.mean
-        markdown += f"{i}. **{framework}** ({cache_status}): {avg_time:.4f} seconds\n"
+    for i, row in enumerate(sorted_stats.to_dict("records"), 1):
+        markdown += (
+            f"{i}. **{row['framework']}** ({row['cache_status']}): "
+            f"{row['mean']:.4f} seconds\n"
+        )
 
     return markdown
 

README.md

@@ -11,7 +11,9 @@ This readme does not mention which tool performed the best, as the repo is desig
 
 With [uv](https://docs.astral.sh/uv/) installed, run the command `bash run.sh`.
 
-A GitHub Action executes the benchmarks and provides artifacts for analysis, including [pyinstrument](https://pyinstrument.readthedocs.io/en/latest/) reports for detailed analysis.
+A GitHub Action executes the benchmarks and provides artifacts for analysis, including [pyinstrument](https://pyinstrument.readthedocs.io/en/latest/) reports for detailed analysis. It picks the newest Python version for which the latest FireDucks release publishes wheels, since FireDucks motivated this repo.
+
+Note that GitHub automatically disables scheduled workflows after 60 days without repository activity. When that happens the weekly run silently stops and has to be re-enabled manually, either from the Actions tab or with `gh workflow enable benchmark.yml`.
 
 # Other notes:
 I suspected `__pycache__` or some Just-in-Time (JIT) compilation artifact could have an impact on performance. Removing the `__pycache__` folder could help with the former, but other than repeating operations, I don't see a proper way of testing the latter.