diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5d66c42..f78569e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,87 +26,33 @@ jobs: - name: Set up Python run: | - # Automatically detect compatible Python version by testing dependency installation - # Try default latest Python first, then downgrade up to 2 times if needed - - echo "Attempting to find compatible Python version..." - - # Try installing default/latest Python - uv python install - - # Get the initial version that was installed - test_version=$(uv python list | grep -E 'cpython-[0-9]+\.[0-9]+' | grep -v 'pypy' | head -n1 | awk '{print $1}' | sed 's/cpython-//' | cut -d'-' -f1) - - for attempt in 1 2 3; do - echo "" - echo "Attempt $attempt:" - echo "Testing Python $test_version..." - - # Create a temporary virtual environment with the specific Python version - # Use attempt number to avoid path collisions - test_venv="/tmp/test-venv-attempt-$attempt-$$" - uv venv --python "$test_version" "$test_venv" 2>&1 - source "$test_venv/bin/activate" - - # Try to compile requirements (capture output separately for error reporting) - compile_output=$(mktemp) - success=false - if uv pip compile requirements.in > /tmp/requirements-$attempt.txt 2>"$compile_output"; then - # Compile succeeded, now try to sync - if uv pip sync /tmp/requirements-$attempt.txt 2>&1; then - echo "✓ Python $test_version is compatible with all frameworks" - echo "PYTHON_VERSION=$test_version" >> $GITHUB_ENV - success=true - else - echo "✗ Python $test_version: pip sync failed" - fi - else - echo "✗ Python $test_version: dependency compilation failed" - fi - - # Show the error for debugging if failed - if [ "$success" = false ]; then - echo "Error output:" - cat "$compile_output" 2>/dev/null || true - fi - - # Clean up test venv - deactivate 2>/dev/null || true - rm -rf "$test_venv" - rm -f "$compile_output" - - # If successful, we're done - if [ "$success" = true ]; then - break - fi - - # If not the last attempt, try downgrading - if [ $attempt -lt 3 ]; then - # Parse version and downgrade - major=$(echo "$test_version" | cut -d. -f1) - minor=$(echo "$test_version" | cut -d. -f2) - - # Downgrade minor version - new_minor=$((minor - 1)) - - if [ $new_minor -ge 10 ]; then - test_version="$major.$new_minor" - echo "Downgrading to Python $test_version..." - - # Install downgraded version - uv python install "$test_version" - else - echo "Cannot downgrade further (reached Python 3.10)" - exit 1 - fi - else - echo "Error: No compatible Python version found after 3 attempts" - exit 1 - fi - done - - echo "" - echo "Using Python $PYTHON_VERSION" + # fireducks is the framework that lags newest CPython, so its published + # wheels decide the version: take the highest cp3XX Linux wheel tag of + # the latest fireducks release. Auto-adopts 3.14+ once fireducks ships it. + minor=$(curl -sSf --max-time 30 --retry 2 https://pypi.org/pypi/fireducks/json \ + | jq -r '.urls[].filename + | select(test("manylinux")) + | capture("-cp3(?[0-9]+)-").m' \ + | sort -n | tail -1) + + if [ -z "$minor" ]; then + echo "Error: found no manylinux wheel for the latest fireducks release" + exit 1 + fi + + version="3.$minor" + echo "Latest fireducks supports up to Python $version" + + # Confirm every other dependency resolves there too, so an incompatible + # pin fails here with a clear message rather than midway through run.sh. + uv python install "$version" + if ! uv pip compile --python-version "$version" -q requirements.in > /dev/null; then + echo "Error: dependencies do not resolve on Python $version" + exit 1 + fi + + echo "PYTHON_VERSION=$version" >> "$GITHUB_ENV" + echo "Using Python $version" - name: Install system dependencies run: | diff --git a/02_benchmark.py b/02_benchmark.py index 3a9ef01..1aa0446 100644 --- a/02_benchmark.py +++ b/02_benchmark.py @@ -102,11 +102,13 @@ def groupby_aggregation_operation(): time_operation( "complex_multi_join", df_lib, - lambda: orders.merge(customers, on="customer_id") - .merge(order_items, on="order_id") - .merge(products, on="product_id") - .sort_values(["order_id", "order_item_id"]) - .reset_index(drop=True), + lambda: ( + orders.merge(customers, on="customer_id") + .merge(order_items, on="order_id") + .merge(products, on="product_id") + .sort_values(["order_id", "order_item_id"]) + .reset_index(drop=True) + ), ) ) @@ -114,12 +116,14 @@ def groupby_aggregation_operation(): time_operation( "four_table_join", df_lib, - lambda: customers.merge(orders, on="customer_id") - .merge(order_items, on="order_id") - .merge(products, on="product_id") - .merge(reviews, on=["customer_id", "product_id"]) - .sort_values("customer_id") - .reset_index(drop=True), + lambda: ( + customers.merge(orders, on="customer_id") + .merge(order_items, on="order_id") + .merge(products, on="product_id") + .merge(reviews, on=["customer_id", "product_id"]) + .sort_values("customer_id") + .reset_index(drop=True) + ), ) ) @@ -272,10 +276,12 @@ def rolling_operations_func(): time_operation( "conditional_join", df_lib, - lambda: customers.merge(orders, on="customer_id") - .query("age > 25 and total_amount > 100") - .sort_values("customer_id") - .reset_index(drop=True), + lambda: ( + customers.merge(orders, on="customer_id") + .query("age > 25 and total_amount > 100") + .sort_values("customer_id") + .reset_index(drop=True) + ), ) ) @@ -319,9 +325,13 @@ def rolling_operations_func(): time_operation( "time_series_resample", df_lib, - lambda: time_series.set_index("date") - .resample("ME") - .agg({"sales": "sum", "marketing_spend": "sum", "website_visits": "mean"}), + lambda: ( + time_series.set_index("date") + .resample("ME") + .agg( + {"sales": "sum", "marketing_spend": "sum", "website_visits": "mean"} + ) + ), ) ) diff --git a/05_comparison.py b/05_comparison.py index c46ee7f..858bf83 100644 --- a/05_comparison.py +++ b/05_comparison.py @@ -410,7 +410,7 @@ def generate_summary_statistics_markdown(summary_stats: pd.DataFrame) -> str: markdown += "## Framework Performance Overview\n\n" # Convert to markdown table - markdown += summary_stats.to_markdown() + "\n\n" + markdown += f"{summary_stats.to_markdown()}\n\n" # Add some interpretation markdown += "## Key Metrics Explanation\n\n" @@ -429,11 +429,11 @@ def generate_summary_statistics_markdown(summary_stats: pd.DataFrame) -> str: # Reset index to make framework and cache_status regular columns for sorting sorted_stats = summary_stats.reset_index().sort_values("mean") - for i, row in enumerate(sorted_stats.itertuples(), 1): - framework = row.framework - cache_status = row.cache_status - avg_time = row.mean - markdown += f"{i}. **{framework}** ({cache_status}): {avg_time:.4f} seconds\n" + for i, row in enumerate(sorted_stats.to_dict("records"), 1): + markdown += ( + f"{i}. **{row['framework']}** ({row['cache_status']}): " + f"{row['mean']:.4f} seconds\n" + ) return markdown diff --git a/README.md b/README.md index 006ce10..84171e3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ This readme does not mention which tool performed the best, as the repo is desig With [uv](https://docs.astral.sh/uv/) installed, run the command `bash run.sh`. -A GitHub Action executes the benchmarks and provides artifacts for analysis, including [pyinstrument](https://pyinstrument.readthedocs.io/en/latest/) reports for detailed analysis. +A GitHub Action executes the benchmarks and provides artifacts for analysis, including [pyinstrument](https://pyinstrument.readthedocs.io/en/latest/) reports for detailed analysis. It picks the newest Python version for which the latest FireDucks release publishes wheels, since FireDucks motivated this repo. + +Note that GitHub automatically disables scheduled workflows after 60 days without repository activity. When that happens the weekly run silently stops and has to be re-enabled manually, either from the Actions tab or with `gh workflow enable benchmark.yml`. # Other notes: I suspected `__pycache__` or some Just-in-Time (JIT) compilation artifact could have an impact on performance. Removing the `__pycache__` folder could help with the former, but other than repeating operations, I don't see a proper way of testing the latter.