From 9fb2ab80211af06673d57baa8a3a18c8416bdb84 Mon Sep 17 00:00:00 2001 From: Arpandeep Khatua <54747935+akhatua2@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:36:35 -0700 Subject: [PATCH] dataset: verify every feature against combined.patch, on both architectures Second verification pass over all 199 features. Adds the third dataset invariant next to fail-on-base / pass-on-gold: each feature's tests must pass against its task's combined.patch. That is the property the coop eval depends on -- test_merged runs a feature's tests against the merged tree -- and it had never been measured. 19 features in 5 tasks failed it: dspy/8563 f2-f6 f1 changes ToolCalls.format()'s return shape; two pre-existing tests in the shared file asserted the old shape, so every tree containing f1 failed them. 8 of 15 pairs were unwinnable. Sibling tests.patch files now accept both shapes. dspy/8635 f1-f6 f6's min_instr_chars=30 replaces the base tests' 11-char dummy. #40 had set the default to 0 in combined.patch only, contradicting f6's spec and gold. Default restored; sibling assertions accept either value. 5 of 15 pairs were unwinnable. tiktoken/0 f3 combined.patch still used the pre-#46 parameter name. jinja/1559 f3 combined.patch did not implement priority= at all; f3's gold merged in. jinja/1465 all combined.patch carried the PR's test-file diffs and could not apply. No feature.patch changed. Reasoning per feature in dataset/SPEC_AUDIT.md ("Second pass"). Also: three task images (react-hook-form 153/85, huggingface-datasets 3997) had no arm64 manifest; arm64 builds were added next to the unchanged amd64 ones. task85's Dockerfile pins pnpm@8 (unpinned resolves to v10, which refuses to build). scripts/check_combined.py is the new sweep; check_gradeable.py gains --backend docker, chunked sandbox writes (Modal ARG_MAX), and a cached digest lookup (Docker Hub anonymous rate limit). Verified: all 199 pass all three checks on Modal (linux/amd64) and local Docker (linux/arm64). --- .gitignore | 1 + CHANGELOG.md | 31 + dataset/.spec_audit_verdicts.json | 108 +- dataset/README.md | 25 +- dataset/SPEC_AUDIT.md | 95 +- .../dspy_task/task8563/feature2/tests.patch | 23 +- .../dspy_task/task8563/feature3/tests.patch | 23 +- .../dspy_task/task8563/feature4/tests.patch | 24 +- .../dspy_task/task8563/feature5/tests.patch | 24 +- .../dspy_task/task8563/feature6/tests.patch | 24 +- dataset/dspy_task/task8635/combined.patch | 4 +- .../dspy_task/task8635/feature1/tests.patch | 23 +- .../dspy_task/task8635/feature2/tests.patch | 23 +- .../dspy_task/task8635/feature3/tests.patch | 19 +- .../dspy_task/task8635/feature4/tests.patch | 23 +- .../dspy_task/task8635/feature5/tests.patch | 23 +- .../openai_tiktoken_task/task0/combined.patch | 6 +- .../task1465/combined.patch | 1893 ----------------- .../task1559/combined.patch | 288 ++- .../react_hook_form_task/task85/Dockerfile | 5 +- scripts/check_combined.py | 138 ++ scripts/check_gradeable.py | 82 +- 22 files changed, 842 insertions(+), 2063 deletions(-) create mode 100644 scripts/check_combined.py diff --git a/.gitignore b/.gitignore index 0f12bdda..d4a5aa78 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,4 @@ workspace/ # Regenerated by scripts/check_gradeable.py on every run dataset/gradeable_report*.json +dataset/combined_report*.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b9a82f0..5c769d72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Every feature's tests now pass against its task's `combined.patch`** — the third dataset + invariant, alongside fail-on-base / pass-on-gold, and the one the coop eval actually depends on: + `test_merged` runs each feature's `tests.patch` against the merged tree, so a feature whose tests + fail once a sibling's gold is present is unwinnable for every pair containing that sibling. + Swept all 199 on Modal (linux/amd64) and local Docker (linux/arm64): 19 features in 5 tasks failed. + `dspy/8563` f2–f6 (f1 changes `ToolCalls.format()`'s return shape; the pre-existing shape tests + in the shared file failed in any tree with f1 — **8/15 pairs were hard zeros**) and `dspy/8635` + f1–f6 (f6's `min_instr_chars=30` replaces the base tests' 11-char dummy instruction — **5/15 + pairs**; March's #40 had papered over it by setting the default to 0 in combined.patch only, + contradicting f6's spec and gold) are fixed by making the pre-existing assertions in the sibling + `tests.patch` files tolerant of the sibling behaviour; `openai_tiktoken/0` f3 (combined.patch + still used the pre-#46 parameter name) and `pallets_jinja/1559` f3 (combined.patch did not + implement `priority=` at all) and `pallets_jinja/1465` all 10 (combined.patch carried the PR's test-file diffs, so it could not apply on top of any `tests.patch`) are fixed in combined.patch. No `feature.patch` changed. Reasoning + in `dataset/SPEC_AUDIT.md` ("Second pass"). +- Re-verified fail-on-base / pass-on-gold for all 199 features on both architectures: 199/199. +- **Three task images had no linux/arm64 manifest** (`react-hook-form:task153`, `:task85`, + `huggingface-datasets:task3997`), so those 13 features could not run on an arm64 Docker host at + all. arm64 images were built natively and added to the existing indexes alongside the untouched + amd64 manifests. + +### Added + +- **`scripts/check_combined.py`** — the sweep above; one sandbox per feature through the image's + own `runner.sh`, `--backend modal|docker`. +- **`scripts/check_gradeable.py --backend docker`** — run the fail-on-base / pass-on-gold sweep on + the local daemon (arm64 on Apple Silicon) as well as Modal (amd64). Both scripts now write + sandbox files in 32 KB chunks; a single `echo` of `pallets_jinja/1465`'s 107 KB combined.patch + exceeded Modal's 64 KiB `ARG_MAX` and showed up as 10 spurious `ERROR`s. + ## [0.0.29] - 2026-08-14 ### Fixed diff --git a/dataset/.spec_audit_verdicts.json b/dataset/.spec_audit_verdicts.json index 56d6ce6a..59b7f3b5 100644 --- a/dataset/.spec_audit_verdicts.json +++ b/dataset/.spec_audit_verdicts.json @@ -21,7 +21,7 @@ ], "dottxt_ai_outlines_task/task1655|feature10": [ "OK", - "over-complete \u2014 spec contains the answer verbatim" + "over-complete — spec contains the answer verbatim" ], "dottxt_ai_outlines_task/task1655|feature2": [ "OK", @@ -49,7 +49,7 @@ ], "dottxt_ai_outlines_task/task1655|feature8": [ "OK", - "format-only is explicit ('format' x3, XXX-XX-XXXX given) \u2014 note 000-00-0000 and 999-99-9999 are valid, so real SSN area rules must NOT be applied" + "format-only is explicit ('format' x3, XXX-XX-XXXX given) — note 000-00-0000 and 999-99-9999 are valid, so real SSN area rules must NOT be applied" ], "dottxt_ai_outlines_task/task1655|feature9": [ "OK", @@ -57,7 +57,7 @@ ], "dottxt_ai_outlines_task/task1706|feature1": [ "OK", - "'\u2026is not available' is pre-existing outlines text (0 occurrences in gold)" + "'…is not available' is pre-existing outlines text (0 occurrences in gold)" ], "dottxt_ai_outlines_task/task1706|feature2": [ "OK", @@ -73,7 +73,7 @@ ], "dottxt_ai_outlines_task/task1706|feature5": [ "SPEC", - "exact custom_adapter ValueError texts \u2014 conditions stated, wording was not" + "exact custom_adapter ValueError texts — conditions stated, wording was not" ], "dottxt_ai_outlines_task/task1706|feature6": [ "OK", @@ -93,7 +93,7 @@ ], "dspy_task/task8394|feature2": [ "OK", - "over-complete \u2014 gives the exact ns_hash formula and key format" + "over-complete — gives the exact ns_hash formula and key format" ], "dspy_task/task8394|feature3": [ "OK", @@ -112,72 +112,72 @@ "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" ], "dspy_task/task8563|feature2": [ - "OK", - "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" + "TEST", + "pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1)" ], "dspy_task/task8563|feature3": [ - "OK", - "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" + "TEST", + "pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1)" ], "dspy_task/task8563|feature4": [ - "OK", - "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" + "TEST", + "pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1)" ], "dspy_task/task8563|feature5": [ - "OK", - "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" + "TEST", + "pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1)" ], "dspy_task/task8563|feature6": [ - "OK", - "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation" + "TEST", + "pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1)" ], "dspy_task/task8587|feature1": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8587|feature2": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8587|feature3": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8587|feature4": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8587|feature5": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8587|feature6": [ "OK", - "nothing flagged \u2014 streaming API fully named in each spec" + "nothing flagged — streaming API fully named in each spec" ], "dspy_task/task8635|feature1": [ - "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "TEST", + "pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6" ], "dspy_task/task8635|feature2": [ - "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "TEST", + "pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6" ], "dspy_task/task8635|feature3": [ - "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "TEST", + "pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6" ], "dspy_task/task8635|feature4": [ - "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "TEST", + "pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6" ], "dspy_task/task8635|feature5": [ - "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "TEST", + "pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6" ], "dspy_task/task8635|feature6": [ "OK", - "nothing flagged \u2014 proposer params fully named in each spec" + "nothing flagged — proposer params fully named in each spec; combined.patch min_instr_chars default restored to 30 (spec + gold), reverting #40" ], "go_chi_task/task26|feature1": [ "OK", @@ -189,7 +189,7 @@ ], "go_chi_task/task26|feature3": [ "SPEC", - "added RouteSelector API \u2014 spec named no identifiers at all" + "added RouteSelector API — spec named no identifiers at all" ], "go_chi_task/task26|feature4": [ "SPEC", @@ -205,7 +205,7 @@ ], "go_chi_task/task27|feature3": [ "SPEC", - "added EnableDebugLogging() \u2014 gold introduces it, spec named nothing" + "added EnableDebugLogging() — gold introduces it, spec named nothing" ], "go_chi_task/task27|feature4": [ "OK", @@ -237,7 +237,7 @@ ], "huggingface_datasets_task/task3997|feature2": [ "SPEC", - "added set_custom_decoding_criteria \u2014 gold introduces it, spec named nothing" + "added set_custom_decoding_criteria — gold introduces it, spec named nothing" ], "huggingface_datasets_task/task3997|feature3": [ "SPEC", @@ -329,7 +329,7 @@ ], "llama_index_task/task18813|feature2": [ "SPEC", - "exact 'exceeds maximum allowed size' message \u2014 spec gave only the condition" + "exact 'exceeds maximum allowed size' message — spec gave only the condition" ], "llama_index_task/task18813|feature3": [ "OK", @@ -361,7 +361,7 @@ ], "openai_tiktoken_task/task0|feature3": [ "OK", - "encode() parameter named with its post-processing semantics stated" + "encode() parameter named with its post-processing semantics stated; combined.patch renamed return_frequency -> analyze_frequency to match spec/gold (#46 missed it)" ], "openai_tiktoken_task/task0|feature4": [ "OK", @@ -381,7 +381,7 @@ ], "openai_tiktoken_task/task0|feature8": [ "OK", - "recomputed the dict \u2014 exactly \"top 3 adjacent pairs\"" + "recomputed the dict — exactly \"top 3 adjacent pairs\"" ], "openai_tiktoken_task/task0|feature9": [ "OK", @@ -405,7 +405,7 @@ ], "pallets_click_task/task2068|feature2": [ "SPEC", - "exact timeout message \u2014 and gold does not pluralise, so \"after 1 seconds\"" + "exact timeout message — and gold does not pluralise, so \"after 1 seconds\"" ], "pallets_click_task/task2068|feature3": [ "SPEC", @@ -437,7 +437,7 @@ ], "pallets_click_task/task2800|feature1": [ "OK", - "over-complete \u2014 names function, call sites, construct" + "over-complete — names function, call sites, construct" ], "pallets_click_task/task2800|feature2": [ "OK", @@ -549,11 +549,11 @@ ], "pallets_jinja_task/task1559|feature3": [ "OK", - "i18n extension API and tag syntax fully named in the spec" + "i18n extension API and tag syntax fully named in the spec; combined.patch gained the priority= implementation, which it did not contain at all" ], "pallets_jinja_task/task1559|feature4": [ "OK", - "over-complete \u2014 error strings quoted verbatim" + "over-complete — error strings quoted verbatim" ], "pallets_jinja_task/task1559|feature5": [ "OK", @@ -677,47 +677,47 @@ ], "react_hook_form_task/task153|feature1": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task153|feature2": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task153|feature3": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task153|feature4": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task153|feature5": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task153|feature6": [ "OK", - "over-complete \u2014 callback signature, ordering and target file all stated" + "over-complete — callback signature, ordering and target file all stated" ], "react_hook_form_task/task85|feature1": [ "OK", - "over-complete \u2014 names the prop, its type, the file and often the implementation shape" + "over-complete — names the prop, its type, the file and often the implementation shape" ], "react_hook_form_task/task85|feature2": [ "OK", - "over-complete \u2014 names the prop, its type, the file and often the implementation shape" + "over-complete — names the prop, its type, the file and often the implementation shape" ], "react_hook_form_task/task85|feature3": [ "OK", - "over-complete \u2014 names the prop, its type, the file and often the implementation shape" + "over-complete — names the prop, its type, the file and often the implementation shape" ], "react_hook_form_task/task85|feature4": [ "OK", - "over-complete \u2014 names the prop, its type, the file and often the implementation shape" + "over-complete — names the prop, its type, the file and often the implementation shape" ], "react_hook_form_task/task85|feature5": [ "OK", - "over-complete \u2014 names the prop, its type, the file and often the implementation shape" + "over-complete — names the prop, its type, the file and often the implementation shape" ], "samuelcolvin_dirty_equals_task/task43|feature1": [ "OK", @@ -737,7 +737,7 @@ ], "samuelcolvin_dirty_equals_task/task43|feature5": [ "SPEC", - "country codes are case-insensitive \u2014 gold does .upper(), spec silent" + "country codes are case-insensitive — gold does .upper(), spec silent" ], "samuelcolvin_dirty_equals_task/task43|feature6": [ "SPEC", diff --git a/dataset/README.md b/dataset/README.md index 8d1d421e..08663be6 100644 --- a/dataset/README.md +++ b/dataset/README.md @@ -19,9 +19,12 @@ size_categories: This dataset contains the benchmark tasks for evaluating multi-agent coordination in code collaboration. -**Paper**: [CooperBench: Why Coding Agents Cannot be Your Teammates Yet](https://arxiv.org/abs/2601.13295) -**Code**: [github.com/cooperbench/CooperBench](https://github.com/cooperbench/CooperBench) -**Website**: [cooperbench.com](https://cooperbench.com) +**Run it with the official harness:** [github.com/cooperbench/CooperBench](https://github.com/cooperbench/CooperBench) (`pip install cooperbench`, then `cooperbench prepare` downloads this dataset). The harness, the Docker images, and this dataset are versioned together — use the GitHub repo's `dataset/` tree or this mirror at the matching tag. + +- **Paper**: [CooperBench: Why Coding Agents Cannot be Your Teammates Yet](https://arxiv.org/abs/2601.13295) +- **Code**: [github.com/cooperbench/CooperBench](https://github.com/cooperbench/CooperBench) +- **Website**: [cooperbench.com](https://cooperbench.com) +- **Task images**: `akhatua/cooperbench-:task` on Docker Hub, multi-arch (linux/amd64 + linux/arm64) ## Structure @@ -42,6 +45,20 @@ dataset/ ``` +## Patches: what each one is for + +Each task is one real pull request split into N independent features. The eval never merges gold patches — it tests **an agent's** patch (or the merge of two agents' patches) against each feature's hidden tests. The three patch files mean different things: + +| file | what it is | property it must satisfy | +|---|---|---| +| `feature{N}/feature.patch` | the gold implementation of feature N **alone** | passes `feature{N}/tests.patch` on the base commit | +| `feature{N}/tests.patch` | feature N's hidden tests | **fails** on the untouched base commit | +| `combined.patch` | the whole PR — every feature landed in one tree, conflicts already resolved | passes **every** feature's `tests.patch` | + +**Gold patches for two different features are not meant to be merged.** They were carved out of the same PR and edit the same files, so `git merge` of `feature1.patch` and `feature2.patch` conflicts for 499 of the 652 pairs (see `gold_conflict_report.json`). That is the coordination challenge the benchmark measures, not a defect: two agents each implementing one feature must produce patches that *do* merge. The resolved oracle for any pair is `combined.patch` — it contains both features and passes both test suites. + +All three properties are verified for all 199 features on both architectures with the harness's `scripts/check_gradeable.py` and `scripts/check_combined.py`; every fix and its reasoning is logged in [`SPEC_AUDIT.md`](SPEC_AUDIT.md). + ## Repositories | Directory | Repository | Tasks | Features | @@ -57,7 +74,7 @@ dataset/ | `pillow_task` | [python-pillow/Pillow](https://github.com/python-pillow/Pillow) | 3 | 15 | | `react_hook_form_task` | [react-hook-form/react-hook-form](https://github.com/react-hook-form/react-hook-form) | 2 | 11 | | `samuelcolvin_dirty_equals_task` | [samuelcolvin/dirty-equals](https://github.com/samuelcolvin/dirty-equals) | 1 | 9 | -| `typst` | [typst/typst](https://github.com/typst/typst) | 1 | 10 | +| `typst_task` | [typst/typst](https://github.com/typst/typst) | 1 | 10 | ## Subsets diff --git a/dataset/SPEC_AUDIT.md b/dataset/SPEC_AUDIT.md index 5112d4d7..509fc1b1 100644 --- a/dataset/SPEC_AUDIT.md +++ b/dataset/SPEC_AUDIT.md @@ -47,7 +47,7 @@ reasoner covers it. **Verdict: derivable, no change.** ## Status — 199/199 features audited -`OK` 174 · `SPEC` 23 · `TEST` 2 +`OK` 164 · `SPEC` 23 · `TEST` 12 `OK` sufficient, no change · `SPEC` feature.md amended · `TEST` tests.patch amended · `RUN?` defect identified, fix needs a container run · `-` not yet audited @@ -81,23 +81,23 @@ reasoner covers it. **Verdict: derivable, no change.** | | f4 | `OK ` | both candidate gaps derivable; amendment reverted | | | f5 | `OK ` | compression + magic-header format stated in the spec | | dspy_task/task8563 | f1 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | -| | f2 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | -| | f3 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | -| | f4 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | -| | f5 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | -| | f6 | `OK ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation | +| | f2 | `TEST` | pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1) | +| | f3 | `TEST` | pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1) | +| | f4 | `TEST` | pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1) | +| | f5 | `TEST` | pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1) | +| | f6 | `TEST` | pre-existing format() tests made shape-tolerant: f1 switches format() to the compact dict shape, which failed these tests in any tree containing f1 (combined, or any coop pair with f1) | | dspy_task/task8587 | f1 | `OK ` | nothing flagged — streaming API fully named in each spec | | | f2 | `OK ` | nothing flagged — streaming API fully named in each spec | | | f3 | `OK ` | nothing flagged — streaming API fully named in each spec | | | f4 | `OK ` | nothing flagged — streaming API fully named in each spec | | | f5 | `OK ` | nothing flagged — streaming API fully named in each spec | | | f6 | `OK ` | nothing flagged — streaming API fully named in each spec | -| dspy_task/task8635 | f1 | `OK ` | nothing flagged — proposer params fully named in each spec | -| | f2 | `OK ` | nothing flagged — proposer params fully named in each spec | -| | f3 | `OK ` | nothing flagged — proposer params fully named in each spec | -| | f4 | `OK ` | nothing flagged — proposer params fully named in each spec | -| | f5 | `OK ` | nothing flagged — proposer params fully named in each spec | -| | f6 | `OK ` | nothing flagged — proposer params fully named in each spec | +| dspy_task/task8635 | f1 | `TEST` | pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6 | +| | f2 | `TEST` | pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6 | +| | f3 | `TEST` | pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6 | +| | f4 | `TEST` | pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6 | +| | f5 | `TEST` | pre-existing proposer tests accept the signature-default instruction too: f6's min_instr_chars=30 replaces the 11-char dummy proposal, which failed these tests in any tree containing f6 | +| | f6 | `OK ` | nothing flagged — proposer params fully named in each spec; combined.patch min_instr_chars default restored to 30 (spec + gold), reverting #40 | | go_chi_task/task26 | f1 | `OK ` | "alias for chi.URLParam" carries the ordering requirement | | | f2 | `SPEC` | added RouteMetric/SimpleMetricsCollector field names | | | f3 | `SPEC` | added RouteSelector API — spec named no identifiers at all | @@ -143,7 +143,7 @@ reasoner covers it. **Verdict: derivable, no change.** | openai_tiktoken_task/task0 | f1 | `OK ` | encode() parameter named with its post-processing semantics stated | | | f10 | `OK ` | encode() parameter named with its post-processing semantics stated | | | f2 | `OK ` | encode() parameter named with its post-processing semantics stated | -| | f3 | `OK ` | encode() parameter named with its post-processing semantics stated | +| | f3 | `OK ` | encode() parameter named with its post-processing semantics stated; combined.patch renamed return_frequency -> analyze_frequency to match spec/gold (#46 missed it) | | | f4 | `OK ` | encode() parameter named with its post-processing semantics stated | | | f5 | `OK ` | encode() parameter named with its post-processing semantics stated | | | f6 | `OK ` | composition order stated with an example | @@ -190,7 +190,7 @@ reasoner covers it. **Verdict: derivable, no change.** | pallets_jinja_task/task1559 | f1 | `OK ` | i18n extension API and tag syntax fully named in the spec | | | f10 | `OK ` | i18n extension API and tag syntax fully named in the spec | | | f2 | `OK ` | i18n extension API and tag syntax fully named in the spec | -| | f3 | `OK ` | i18n extension API and tag syntax fully named in the spec | +| | f3 | `OK ` | i18n extension API and tag syntax fully named in the spec; combined.patch gained the priority= implementation, which it did not contain at all | | | f4 | `OK ` | over-complete — error strings quoted verbatim | | | f5 | `OK ` | i18n extension API and tag syntax fully named in the spec | | | f6 | `OK ` | i18n extension API and tag syntax fully named in the spec | @@ -878,3 +878,70 @@ Nothing blocking. Every one of the 199 features has been measured base-fails / g The one item deliberately left unfixed is `pillow/290` f4's reference threshold semantics (above) — gradeability holds, but the reference does not honour its own `error_threshold`, and correcting it would mean editing `feature.patch`. + +--- + +# Second pass (2026-09-05) — the third invariant + +The first pass established two properties per feature and re-verified them here on both +published architectures (Modal, linux/amd64; local Docker on Apple Silicon, linux/arm64): + + tests alone -> FAIL 199/199 + tests + feature.patch -> PASS 199/199 + +This pass adds the third, which nothing had ever measured: + + tests + combined.patch -> PASS `scripts/check_combined.py` + +`combined.patch` is the whole PR, i.e. the reference for "every feature landed in one tree". If a +feature's tests fail against it, that feature's tests cannot pass in *any* tree that also contains +the sibling feature responsible — and the coop eval runs each feature's `tests.patch` against the +merged tree (`eval/sandbox.py::test_merged`), so the pair is unwinnable no matter what the agents +do. That is the property that actually matters for the benchmark; the combined patch is just the +cheapest way to test it for all siblings at once. + +**Result: 180/199 passed; 19 failures in 5 tasks.** (10 of the 19 first surfaced as harness +`ERROR`s: `pallets_jinja/1465`'s 107 KB combined.patch exceeded Modal's 64 KiB `ARG_MAX` in the +checker's file write. Chunking the write showed all 10 were real failures too.) + +| task | features | cause | fix | +|---|---|---|---| +| `dspy/8563` | f2 f3 f4 f5 f6 | f1's gold changes `ToolCalls.format()` from `[{"type": "tool_calls", ...}]` to `{"tool_calls": [...]}`. The two pre-existing tests asserting the old shape live in `tests/adapters/test_tool.py`, which every feature's runner executes in full — so they fail in every tree containing f1. f1's own `tests.patch` rewrites them; the others carried the originals. | `tests.patch` of f2–f6: the two pre-existing tests normalise a dict result to the list shape before asserting. Coverage unchanged; each still fails on base and passes on its own gold. **8 of the task's 15 pairs (every pair containing f1) were unwinnable before this.** | +| `dspy/8635` | f6 (and f1–f5 latently) | f6's spec and gold set `min_instr_chars=30`; the base tests propose an 11-char `"instruction"` and assert it comes back verbatim, which f6 correctly replaces with the signature default. #40 (March) "fixed" this by setting the default to **0 in combined.patch only** — which silenced f1–f5 against combined but made combined contradict f6's spec and gold. | combined.patch default restored to 30 (revert of #40). `tests.patch` of f1–f5: the two pre-existing assertions accept either the verbatim proposal or the signature default. **5 of 15 pairs (every pair containing f6) were unwinnable before this.** | +| `openai_tiktoken/0` | f3 | #46 renamed the parameter `return_frequency → analyze_frequency` in f3's gold and tests to match the spec, but not in combined.patch. | combined.patch renamed to match. Pure consistency; no coop pair was affected because no sibling touches that parameter. | +| `pallets_jinja/1465` | all 10 | combined.patch carried the PR's **test-file** diffs (`tests/test_filters.py`, `tests/test_async_filters.py`) alongside `src/jinja2/filters.py`. The runner applies `tests.patch` first, so combined then failed to apply for every feature. The only combined.patch in the dataset that touched a test path. | test-file diffs stripped from combined.patch; source diff untouched. All 10 features pass against it. Consistency only — the eval never applies combined.patch. | +| `pallets_jinja/1559` | f3 | combined.patch did not implement `{% trans priority=N %}` at all — it carried a different design (`_make_new_gettext_with_fallback`) from the original PR, while f3's spec, gold and tests describe the priority retry. Zero occurrences of `priority` in combined. | combined.patch regenerated: the combined tree with f3's gold hunks merged in by hand (five conflicting regions in `parse()` / `_make_node()`, resolved so `priority=` is parsed alongside `domain=` and the metadata parameters). All 10 features pass against it. | + +No `feature.patch` was touched. Where a fix had a choice it went to `tests.patch` (loosening a +pre-existing assertion that was never about the feature) or to `combined.patch` (which is a +derived artefact of the gold patches and is not read by the eval at all — `grep combined src/` +finds nothing). + +Two of the five are a single pattern worth naming: **a feature that changes pre-existing behaviour +breaks the pre-existing tests, and every sibling's `tests.patch` inherits those tests because the +runners execute whole files.** The first pass noted this as a source of noise ("a feature's score +depends on its partner not breaking anything in the same file"); this pass shows it was not noise +but a hard zero for 13 pairs. `check_combined.py` is the regression test for it. + +Two transients, both confirmed by re-runs on Modal and locally (3/3): `go_chi/56` f1 failed once +against combined with no test output; `dspy/8587` f4 failed once with two +`TimeoutError: Server on port N did not become ready within 10 seconds` errors. The latter comes +from dspy's own `tests/test_utils/server/__init__.py`, which the runner pulls in via +`tests/streaming/test_streaming.py` — a 10 s fixed readiness wait that a loaded sandbox can miss. +It is pre-existing test infrastructure, not part of any `tests.patch`, so it was left alone, but +it is a known source of false negatives for every feature of that task under heavy eval +concurrency. + +## Architecture coverage of the published images + +The first pass documented the images as multi-arch (linux/amd64 + linux/arm64) and warned that a +plain `docker build` on Apple Silicon would break amd64. The inverse was never checked: on +2026-09-05, 3 of the 30 tags had **no arm64 manifest at all** — `react-hook-form:task153`, +`react-hook-form:task85`, `huggingface-datasets:task3997` — so those 3 tasks (13 features, 35 +pairs) could not run on any arm64 Docker host, and every local sweep on a Mac would have failed +them with `no matching manifest for linux/arm64/v8`. The other 27 were fine. + +Fixed without touching amd64: the arm64 image is built natively from the task's Dockerfile and +stitched into the existing index next to the *unchanged* amd64 manifest +(`docker buildx imagetools create -t @ -arm64`), so the bytes every +Modal result above was measured against are the same bytes. diff --git a/dataset/dspy_task/task8563/feature2/tests.patch b/dataset/dspy_task/task8563/feature2/tests.patch index 3e52cab9..bb51d3cf 100644 --- a/dataset/dspy_task/task8563/feature2/tests.patch +++ b/dataset/dspy_task/task8563/feature2/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py -index 185714539..cc2b3609e 100644 +index 185714539..8a8b5f9d5 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,7 @@ import pytest @@ -11,7 +11,15 @@ index 185714539..cc2b3609e 100644 # Test fixtures -@@ -444,6 +444,81 @@ def test_tool_calls_format_basic(tool_calls_data, expected): +@@ -440,10 +440,89 @@ def test_tool_calls_format_basic(tool_calls_data, expected): + tool_calls_list = [ToolCalls.ToolCall(**data) for data in tool_calls_data] + tool_calls = ToolCalls(tool_calls=tool_calls_list) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + assert result == expected @@ -93,3 +101,14 @@ index 185714539..cc2b3609e 100644 def test_tool_calls_format_from_dict_list(): """Test format works with ToolCalls created from from_dict_list.""" tool_calls_dicts = [ +@@ -453,6 +532,10 @@ def test_tool_calls_format_from_dict_list(): + + tool_calls = ToolCalls.from_dict_list(tool_calls_dicts) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert len(result[0]["tool_calls"]) == 2 + assert result[0]["tool_calls"][0]["function"]["name"] == "search" diff --git a/dataset/dspy_task/task8563/feature3/tests.patch b/dataset/dspy_task/task8563/feature3/tests.patch index decbc343..b3f0f286 100644 --- a/dataset/dspy_task/task8563/feature3/tests.patch +++ b/dataset/dspy_task/task8563/feature3/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py -index 185714539..cb2a8d968 100644 +index 185714539..30892e648 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,7 @@ import pytest @@ -11,7 +11,26 @@ index 185714539..cb2a8d968 100644 # Test fixtures -@@ -457,3 +457,115 @@ def test_tool_calls_format_from_dict_list(): +@@ -440,6 +440,10 @@ def test_tool_calls_format_basic(tool_calls_data, expected): + tool_calls_list = [ToolCalls.ToolCall(**data) for data in tool_calls_data] + tool_calls = ToolCalls(tool_calls=tool_calls_list) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert result == expected + +@@ -453,7 +457,123 @@ def test_tool_calls_format_from_dict_list(): + + tool_calls = ToolCalls.from_dict_list(tool_calls_dicts) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + assert len(result[0]["tool_calls"]) == 2 assert result[0]["tool_calls"][0]["function"]["name"] == "search" assert result[0]["tool_calls"][1]["function"]["name"] == "translate" diff --git a/dataset/dspy_task/task8563/feature4/tests.patch b/dataset/dspy_task/task8563/feature4/tests.patch index 31d02fba..f1ba5e32 100644 --- a/dataset/dspy_task/task8563/feature4/tests.patch +++ b/dataset/dspy_task/task8563/feature4/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py -index 185714539..d4927bfee 100644 +index 185714539..78d46dfbc 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,7 @@ import pytest @@ -167,3 +167,25 @@ index 185714539..d4927bfee 100644 def test_tool_from_function_with_pydantic_nesting(): tool = Tool(complex_dummy_function) +@@ -440,6 +589,10 @@ def test_tool_calls_format_basic(tool_calls_data, expected): + tool_calls_list = [ToolCalls.ToolCall(**data) for data in tool_calls_data] + tool_calls = ToolCalls(tool_calls=tool_calls_list) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert result == expected + +@@ -453,6 +606,10 @@ def test_tool_calls_format_from_dict_list(): + + tool_calls = ToolCalls.from_dict_list(tool_calls_dicts) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert len(result[0]["tool_calls"]) == 2 + assert result[0]["tool_calls"][0]["function"]["name"] == "search" diff --git a/dataset/dspy_task/task8563/feature5/tests.patch b/dataset/dspy_task/task8563/feature5/tests.patch index e5a1d626..1eb00fd7 100644 --- a/dataset/dspy_task/task8563/feature5/tests.patch +++ b/dataset/dspy_task/task8563/feature5/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py -index 185714539..bda5fe91e 100644 +index 185714539..cecde433e 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,7 @@ import pytest @@ -274,3 +274,25 @@ index 185714539..bda5fe91e 100644 def test_tool_with_default_args_without_type_hints(): def foo(x=100): return x +@@ -440,6 +696,10 @@ def test_tool_calls_format_basic(tool_calls_data, expected): + tool_calls_list = [ToolCalls.ToolCall(**data) for data in tool_calls_data] + tool_calls = ToolCalls(tool_calls=tool_calls_list) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert result == expected + +@@ -453,6 +713,10 @@ def test_tool_calls_format_from_dict_list(): + + tool_calls = ToolCalls.from_dict_list(tool_calls_dicts) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert len(result[0]["tool_calls"]) == 2 + assert result[0]["tool_calls"][0]["function"]["name"] == "search" diff --git a/dataset/dspy_task/task8563/feature6/tests.patch b/dataset/dspy_task/task8563/feature6/tests.patch index 6bccd5f1..a8a04288 100644 --- a/dataset/dspy_task/task8563/feature6/tests.patch +++ b/dataset/dspy_task/task8563/feature6/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py -index 185714539..4e849662c 100644 +index 185714539..bdc1583d8 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,7 @@ import pytest @@ -141,3 +141,25 @@ index 185714539..4e849662c 100644 def complex_dummy_function(profile: UserProfile, priority: int, notes: list[Note] | None = None) -> dict[str, Any]: """Process user profile with complex nested structure. +@@ -440,6 +563,10 @@ def test_tool_calls_format_basic(tool_calls_data, expected): + tool_calls_list = [ToolCalls.ToolCall(**data) for data in tool_calls_data] + tool_calls = ToolCalls(tool_calls=tool_calls_list) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert result == expected + +@@ -453,6 +580,10 @@ def test_tool_calls_format_from_dict_list(): + + tool_calls = ToolCalls.from_dict_list(tool_calls_dicts) + result = tool_calls.format() ++ if isinstance(result, dict): ++ # Another feature of this PR may switch format() to the compact {"tool_calls": [...]} shape; ++ # this test is about the payload, so accept both shapes. ++ result = [{"type": "tool_calls", **result}] + + assert len(result[0]["tool_calls"]) == 2 + assert result[0]["tool_calls"][0]["function"]["name"] == "search" diff --git a/dataset/dspy_task/task8635/combined.patch b/dataset/dspy_task/task8635/combined.patch index 7ac9f18c..d19704f0 100644 --- a/dataset/dspy_task/task8635/combined.patch +++ b/dataset/dspy_task/task8635/combined.patch @@ -51,7 +51,7 @@ index 13722b4b..2d60a9b6 100644 use_tip=True, verbose=False, + max_description_chars=2000, -+ min_instr_chars=0, ++ min_instr_chars=30, + max_instr_chars=600, + rephrase_when_too_long=False, ): @@ -201,7 +201,7 @@ index 13722b4b..2d60a9b6 100644 + max_instruct_history=5, + tip_weights=None, + max_description_chars=2000, -+ min_instr_chars=0, ++ min_instr_chars=30, + max_instr_chars=600, + rephrase_when_too_long=False, ): diff --git a/dataset/dspy_task/task8635/feature1/tests.patch b/dataset/dspy_task/task8635/feature1/tests.patch index ef7872e5..d10db056 100644 --- a/dataset/dspy_task/task8635/feature1/tests.patch +++ b/dataset/dspy_task/task8635/feature1/tests.patch @@ -1,6 +1,27 @@ +diff --git a/tests/propose/test_grounded_proposer.py b/tests/propose/test_grounded_proposer.py +index 252afe8ad..19200a2cf 100644 +--- a/tests/propose/test_grounded_proposer.py ++++ b/tests/propose/test_grounded_proposer.py +@@ -26,7 +26,9 @@ def test_propose_instructions_for_program(demo_candidates): + assert isinstance(result, dict) + assert len(result) == len(program.predictors()) + for pred_instructions in result.values(): +- assert pred_instructions == ["instruction"] ++ # Another feature of this PR (instruction length bounds) may replace a too-short proposal ++ # with the signature's default instruction; this test is not about that, so accept both. ++ assert pred_instructions in (["instruction"], ["Given the fields `question`, produce the fields `answer`."]) + + + @pytest.mark.parametrize( +@@ -51,4 +53,4 @@ def test_propose_instruction_for_predictor(demo_candidates): + trial_logs={}, + tip=None, + ) +- assert result == "instruction" ++ assert result in ("instruction", "Given the fields `question`, produce the fields `answer`.") diff --git a/tests/propose/test_grounded_proposer1.py b/tests/propose/test_grounded_proposer1.py new file mode 100644 -index 00000000..3a735698 +index 000000000..3a735698a --- /dev/null +++ b/tests/propose/test_grounded_proposer1.py @@ -0,0 +1,204 @@ diff --git a/dataset/dspy_task/task8635/feature2/tests.patch b/dataset/dspy_task/task8635/feature2/tests.patch index ecb4997e..122f040d 100644 --- a/dataset/dspy_task/task8635/feature2/tests.patch +++ b/dataset/dspy_task/task8635/feature2/tests.patch @@ -1,6 +1,27 @@ +diff --git a/tests/propose/test_grounded_proposer.py b/tests/propose/test_grounded_proposer.py +index 252afe8ad..19200a2cf 100644 +--- a/tests/propose/test_grounded_proposer.py ++++ b/tests/propose/test_grounded_proposer.py +@@ -26,7 +26,9 @@ def test_propose_instructions_for_program(demo_candidates): + assert isinstance(result, dict) + assert len(result) == len(program.predictors()) + for pred_instructions in result.values(): +- assert pred_instructions == ["instruction"] ++ # Another feature of this PR (instruction length bounds) may replace a too-short proposal ++ # with the signature's default instruction; this test is not about that, so accept both. ++ assert pred_instructions in (["instruction"], ["Given the fields `question`, produce the fields `answer`."]) + + + @pytest.mark.parametrize( +@@ -51,4 +53,4 @@ def test_propose_instruction_for_predictor(demo_candidates): + trial_logs={}, + tip=None, + ) +- assert result == "instruction" ++ assert result in ("instruction", "Given the fields `question`, produce the fields `answer`.") diff --git a/tests/propose/test_grounded_proposer2.py b/tests/propose/test_grounded_proposer2.py new file mode 100644 -index 00000000..d69e08e9 +index 000000000..d69e08e9c --- /dev/null +++ b/tests/propose/test_grounded_proposer2.py @@ -0,0 +1,91 @@ diff --git a/dataset/dspy_task/task8635/feature3/tests.patch b/dataset/dspy_task/task8635/feature3/tests.patch index ad471bed..f724f3a1 100644 --- a/dataset/dspy_task/task8635/feature3/tests.patch +++ b/dataset/dspy_task/task8635/feature3/tests.patch @@ -1,5 +1,5 @@ diff --git a/tests/propose/test_grounded_proposer.py b/tests/propose/test_grounded_proposer.py -index 252afe8a..f43ee891 100644 +index 252afe8ad..add5fae90 100644 --- a/tests/propose/test_grounded_proposer.py +++ b/tests/propose/test_grounded_proposer.py @@ -5,6 +5,176 @@ from dspy.predict import Predict @@ -179,3 +179,20 @@ index 252afe8a..f43ee891 100644 @pytest.mark.parametrize( "demo_candidates", +@@ -26,7 +196,9 @@ def test_propose_instructions_for_program(demo_candidates): + assert isinstance(result, dict) + assert len(result) == len(program.predictors()) + for pred_instructions in result.values(): +- assert pred_instructions == ["instruction"] ++ # Another feature of this PR (instruction length bounds) may replace a too-short proposal ++ # with the signature's default instruction; this test is not about that, so accept both. ++ assert pred_instructions in (["instruction"], ["Given the fields `question`, produce the fields `answer`."]) + + + @pytest.mark.parametrize( +@@ -51,4 +223,4 @@ def test_propose_instruction_for_predictor(demo_candidates): + trial_logs={}, + tip=None, + ) +- assert result == "instruction" ++ assert result in ("instruction", "Given the fields `question`, produce the fields `answer`.") diff --git a/dataset/dspy_task/task8635/feature4/tests.patch b/dataset/dspy_task/task8635/feature4/tests.patch index 2e13bc51..412d2829 100644 --- a/dataset/dspy_task/task8635/feature4/tests.patch +++ b/dataset/dspy_task/task8635/feature4/tests.patch @@ -1,6 +1,27 @@ +diff --git a/tests/propose/test_grounded_proposer.py b/tests/propose/test_grounded_proposer.py +index 252afe8ad..19200a2cf 100644 +--- a/tests/propose/test_grounded_proposer.py ++++ b/tests/propose/test_grounded_proposer.py +@@ -26,7 +26,9 @@ def test_propose_instructions_for_program(demo_candidates): + assert isinstance(result, dict) + assert len(result) == len(program.predictors()) + for pred_instructions in result.values(): +- assert pred_instructions == ["instruction"] ++ # Another feature of this PR (instruction length bounds) may replace a too-short proposal ++ # with the signature's default instruction; this test is not about that, so accept both. ++ assert pred_instructions in (["instruction"], ["Given the fields `question`, produce the fields `answer`."]) + + + @pytest.mark.parametrize( +@@ -51,4 +53,4 @@ def test_propose_instruction_for_predictor(demo_candidates): + trial_logs={}, + tip=None, + ) +- assert result == "instruction" ++ assert result in ("instruction", "Given the fields `question`, produce the fields `answer`.") diff --git a/tests/propose/test_grounded_proposer4.py b/tests/propose/test_grounded_proposer4.py new file mode 100644 -index 00000000..3d13ee04 +index 000000000..3d13ee049 --- /dev/null +++ b/tests/propose/test_grounded_proposer4.py @@ -0,0 +1,123 @@ diff --git a/dataset/dspy_task/task8635/feature5/tests.patch b/dataset/dspy_task/task8635/feature5/tests.patch index 9e1fb62b..16522962 100644 --- a/dataset/dspy_task/task8635/feature5/tests.patch +++ b/dataset/dspy_task/task8635/feature5/tests.patch @@ -1,6 +1,27 @@ +diff --git a/tests/propose/test_grounded_proposer.py b/tests/propose/test_grounded_proposer.py +index 252afe8ad..19200a2cf 100644 +--- a/tests/propose/test_grounded_proposer.py ++++ b/tests/propose/test_grounded_proposer.py +@@ -26,7 +26,9 @@ def test_propose_instructions_for_program(demo_candidates): + assert isinstance(result, dict) + assert len(result) == len(program.predictors()) + for pred_instructions in result.values(): +- assert pred_instructions == ["instruction"] ++ # Another feature of this PR (instruction length bounds) may replace a too-short proposal ++ # with the signature's default instruction; this test is not about that, so accept both. ++ assert pred_instructions in (["instruction"], ["Given the fields `question`, produce the fields `answer`."]) + + + @pytest.mark.parametrize( +@@ -51,4 +53,4 @@ def test_propose_instruction_for_predictor(demo_candidates): + trial_logs={}, + tip=None, + ) +- assert result == "instruction" ++ assert result in ("instruction", "Given the fields `question`, produce the fields `answer`.") diff --git a/tests/propose/test_grounded_proposer5.py b/tests/propose/test_grounded_proposer5.py new file mode 100644 -index 00000000..7141b053 +index 000000000..7141b053b --- /dev/null +++ b/tests/propose/test_grounded_proposer5.py @@ -0,0 +1,125 @@ diff --git a/dataset/openai_tiktoken_task/task0/combined.patch b/dataset/openai_tiktoken_task/task0/combined.patch index 05fa4b5a..4f906e2e 100644 --- a/dataset/openai_tiktoken_task/task0/combined.patch +++ b/dataset/openai_tiktoken_task/task0/combined.patch @@ -170,7 +170,7 @@ index 6bc9736..a15c50c 100644 - ) -> list[int]: + max_tokens: int | None = None, + return_positions: bool = False, -+ return_frequency: bool = False, ++ analyze_frequency: bool = False, + chunk_size: int | None = None, + filter_tokens: list[int] | None = None, + transformers: Sequence[Callable[[str], str]] | None = None, @@ -238,7 +238,7 @@ index 6bc9736..a15c50c 100644 + and not transformers + and not compression + and not return_positions -+ and not return_frequency ++ and not analyze_frequency + and not return_repeated_pattern + and max_tokens is None + ) @@ -302,7 +302,7 @@ index 6bc9736..a15c50c 100644 + if return_positions: + metadata["positions"] = positions or [] + -+ if return_frequency: ++ if analyze_frequency: + metadata["frequency"] = self._count_token_frequency(tokens) + + if return_repeated_pattern: diff --git a/dataset/pallets_jinja_task/task1465/combined.patch b/dataset/pallets_jinja_task/task1465/combined.patch index a9d7f9e6..f3502ef5 100644 --- a/dataset/pallets_jinja_task/task1465/combined.patch +++ b/dataset/pallets_jinja_task/task1465/combined.patch @@ -229,1896 +229,3 @@ index 80ea6504..735b21ad 100644 @pass_environment -diff --git a/tests/test_async_filters.py b/tests/test_async_filters.py -index 5d4f332e..66f1ddcc 100644 ---- a/tests/test_async_filters.py -+++ b/tests/test_async_filters.py -@@ -57,6 +57,26 @@ def test_groupby(env_async, items): - ] - - -+@pytest.mark.parametrize( -+ ("case_sensitive", "expect"), -+ [ -+ (False, "a: 1, 3\nb: 2\n"), -+ (True, "A: 3\na: 1\nb: 2\n"), -+ ], -+) -+def test_groupby_case(env_async, case_sensitive, expect): -+ tmpl = env_async.from_string( -+ "{% for k, vs in data|groupby('k', case_sensitive=cs) %}" -+ "{{ k }}: {{ vs|join(', ', attribute='v') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[{"k": "a", "v": 1}, {"k": "b", "v": 2}, {"k": "A", "v": 3}], -+ cs=case_sensitive, -+ ) -+ assert out == expect -+ -+ - @mark_dualiter("items", lambda: [("a", 1), ("a", 2), ("b", 1)]) - def test_groupby_tuple_index(env_async, items): - tmpl = env_async.from_string( -@@ -94,6 +114,595 @@ def test_groupby_multidot(env_async, articles): - ] - - -+@mark_dualiter( -+ "items", -+ lambda: [ -+ {"foo": 1, "bar": 2}, -+ {"foo": 2, "bar": 3}, -+ {"foo": 1, "bar": 1}, -+ {"foo": 3, "bar": 4}, -+ ], -+) -+def test_groupby_reverse(env_async, items): -+ tmpl = env_async.from_string( -+ """ -+ {%- for grouper, list in items()|groupby('foo', reverse=True) -%} -+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| -+ {%- endfor %}""" -+ ) -+ assert tmpl.render(items=items).split("|") == [ -+ "3: 3, 4", -+ "2: 2, 3", -+ "1: 1, 2: 1, 1", -+ "", -+ ] -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"name": "alice", "score": 95}, -+ {"name": "bob", "score": 87}, -+ {"name": "charlie", "score": 95}, -+ {"name": "david", "score": 72}, -+ ], -+) -+def test_groupby_reverse_numeric(env_async, data): -+ tmpl = env_async.from_string( -+ "{% for score, items in data()|groupby('score', reverse=True) %}" -+ "{{ score }}: {{ items|map(attribute='name')|join(', ') }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ assert out == "95: alice, charlie|87: bob|72: david|" -+ -+ -+def test_groupby_separator(env_async): -+ # Test with double underscore separator (Django-style) -+ tmpl = env_async.from_string( -+ "{% for city, items in users|groupby('profile__city', separator='__') %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "profile": {"city": "NYC"}}, -+ {"name": "bob", "profile": {"city": "LA"}}, -+ {"name": "charlie", "profile": {"city": "NYC"}}, -+ ] -+ ) -+ assert out == "LA: bob\nNYC: alice, charlie\n" -+ -+ -+def test_groupby_separator_with_default(env_async): -+ # Test separator with default value -+ tmpl = env_async.from_string( -+ "{% for city, items in users|groupby('profile__city', default='Unknown', separator='__') %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "profile": {"city": "NYC"}}, -+ {"name": "bob"}, # Missing profile -+ {"name": "charlie", "profile": {}}, # Missing city -+ ] -+ ) -+ assert out == "NYC: alice\nUnknown: bob, charlie\n" -+ -+ -+def make_users_for_max_groups(): -+ return [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie", "city": "SF"}, -+ {"name": "david", "city": "NY"}, -+ {"name": "eve", "city": "LA"}, -+ ] -+ -+ -+@mark_dualiter("users", make_users_for_max_groups) -+def test_groupby_max_groups(env_async, users): -+ """Test max_groups parameter limits the number of groups returned.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', max_groups=2) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ # Should only return first 2 groups (LA and NY, alphabetically sorted) -+ assert out == "LA: bob, eve\nNY: alice, david\n" -+ -+ -+@mark_dualiter( -+ "users", lambda: [{"name": "alice", "city": "NY"}, {"name": "bob", "city": "LA"}] -+) -+def test_groupby_max_groups_zero(env_async, users): -+ """Test max_groups=0 returns empty result.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', max_groups=0) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "" -+ -+ -+@mark_dualiter( -+ "users", lambda: [{"name": "alice", "city": "NY"}, {"name": "bob", "city": "LA"}] -+) -+def test_groupby_max_groups_larger_than_available(env_async, users): -+ """Test max_groups larger than available groups returns all groups.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', max_groups=10) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ -+@mark_dualiter("users", lambda: []) -+def test_groupby_max_groups_empty_input(env_async, users): -+ """Test max_groups with empty input.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', max_groups=2) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "" -+ -+ -+def test_groupby_max_groups_none(env_async): -+ """Test max_groups=None returns all groups (default behavior).""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users|groupby('city', max_groups=none) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie", "city": "SF"}, -+ ] -+ ) -+ assert out == "LA: bob\nNY: alice\nSF: charlie\n" -+ -+ -+@mark_dualiter("users", make_users_for_max_groups) -+def test_groupby_max_groups_single_group(env_async, users): -+ """Test max_groups=1 returns only first group.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', max_groups=1) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ # Should return only LA (first alphabetically) with both users in that group -+ assert out == "LA: bob, eve\n" -+ -+ -+@mark_dualiter("data", lambda: [("a", 1), ("b", 2), ("a", 3), ("c", 4), ("b", 5)]) -+def test_groupby_max_groups_with_tuple_index(env_async, data): -+ """Test max_groups works with tuple indexing.""" -+ tmpl = env_async.from_string( -+ "{% for grouper, items in data()|groupby(0, max_groups=2) %}" -+ "{{ grouper }}: {{ items|map(attribute='1')|join(',') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ # Should return first 2 groups: a, b -+ assert out == "a: 1,3\nb: 2,5\n" -+ -+ -+@mark_dualiter( -+ "users", -+ lambda: [ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve"}, -+ ], -+) -+def test_groupby_include_empty_true(env_async, users): -+ """Test async groupby with include_empty=True (default behavior)""" -+ tmpl = env_async.from_string( -+ "{% for status, items in users()|groupby('status', include_empty=true) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ # Should include empty string, None, and missing attribute groups -+ assert "active: alice, charlie\n" in out -+ assert ": bob\n" in out # empty string group -+ assert "None: dave\n" in out # None group -+ assert "None: eve\n" in out or ": eve\n" in out # missing attribute -+ -+ -+@mark_dualiter( -+ "users", -+ lambda: [ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve", "status": 0}, -+ {"name": "frank"}, -+ ], -+) -+def test_groupby_include_empty_false(env_async, users): -+ """Test async groupby with include_empty=False excludes falsy values""" -+ tmpl = env_async.from_string( -+ "{% for status, items in users()|groupby('status', include_empty=false) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ # Should only include non-falsy values -+ assert out == "active: alice, charlie\n" -+ -+ -+@mark_dualiter( -+ "users", -+ lambda: [ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve"}, -+ ], -+) -+def test_groupby_include_empty_with_default(env_async, users): -+ """Test async groupby with include_empty=False and default value""" -+ tmpl = env_async.from_string( -+ "{% for status, items in users()|groupby('status', default='unknown', include_empty=false) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ # Should include default value but exclude empty/None -+ assert "active: alice, charlie\n" in out -+ assert "unknown: eve\n" in out -+ # Should not include empty string or None groups -+ assert ": bob" not in out -+ assert "None: dave" not in out -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"value": "truthy"}, -+ {"value": ""}, -+ {"value": None}, -+ {"value": 0}, -+ {"value": []}, -+ {"value": {}}, -+ {"value": False}, -+ {"value": "another"}, -+ ], -+) -+def test_groupby_include_empty_edge_cases(env_async, data): -+ """Test async groupby include_empty with various falsy values""" -+ tmpl = env_async.from_string( -+ "{% for val, items in data()|groupby('value', include_empty=false) %}" -+ "{{ val }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ # Should only include truthy values -+ assert "truthy: 1\n" in out -+ assert "another: 1\n" in out -+ # Should not include any falsy values -+ assert ": 1" not in out.replace("truthy: 1", "").replace("another: 1", "") -+ assert "None:" not in out -+ assert "0:" not in out -+ assert "[]:" not in out -+ assert "{}:" not in out -+ assert "False:" not in out -+ -+ -+@mark_dualiter( -+ "users", -+ lambda: [ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": None}, -+ ], -+) -+def test_groupby_include_empty_backward_compatibility(env_async, users): -+ """Test async groupby backward compatibility with include_empty parameter""" -+ # Test without include_empty parameter (should default to True) -+ tmpl1 = env_async.from_string( -+ "{% for status, items in users()|groupby('status') %}" -+ "{{ status }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ # Test with explicit include_empty=True -+ tmpl2 = env_async.from_string( -+ "{% for status, items in users()|groupby('status', include_empty=true) %}" -+ "{{ status }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ -+ out1 = tmpl1.render(users=users) -+ out2 = tmpl2.render(users=users) -+ -+ # Both should produce identical output -+ assert out1 == out2 -+ assert "active: 1\n" in out1 -+ assert ": 1\n" in out1 # empty string -+ assert "None: 1\n" in out1 # None value -+ -+ -+def test_groupby_key_transform(env_async): -+ # Test basic key transformation for async groupby -+ items = [{"foo": 1, "bar": 2}, {"foo": 2, "bar": 3}, {"foo": 1, "bar": 1}] -+ -+ tmpl = env_async.from_string( -+ """ -+ {%- for grouper, list in items|groupby('foo', key_transform=upper_func) -%} -+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| -+ {%- endfor %}""" -+ ) -+ -+ def upper_func(x): -+ return str(x).upper() -+ -+ result = tmpl.render(items=items, upper_func=upper_func) -+ assert result.split("|") == ["1: 1, 2: 1, 1", "2: 2, 3", ""] -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"type": "A", "value": 3}, -+ {"type": "B", "value": 1}, -+ {"type": "A", "value": 1}, -+ {"type": "B", "value": 2}, -+ {"type": "A", "value": 2}, -+ ], -+) -+def test_groupby_stable_basic(env_async, data): -+ """Test basic stable groupby functionality preserves order within groups.""" -+ tmpl = env_async.from_string( -+ "{% for key, items in data()|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ assert out == "A: 3,1,2|B: 1,2|" -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"type": "B", "value": 3}, -+ {"type": "A", "value": 2}, -+ {"type": "B", "value": 1}, -+ {"type": "A", "value": 3}, -+ {"type": "A", "value": 1}, -+ ], -+) -+def test_groupby_stable_vs_default(env_async, data): -+ """Test that stable=True preserves order while default behavior sorts by key.""" -+ # Default behavior (sorts by grouping key first) -+ tmpl_default = env_async.from_string( -+ "{% for key, items in data()|groupby('type') %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ -+ # Stable behavior (preserves original order) -+ tmpl_stable = env_async.from_string( -+ "{% for key, items in data()|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ -+ default_out = tmpl_default.render(data=data) -+ stable_out = tmpl_stable.render(data=data) -+ -+ assert default_out == "A: 2,3,1|B: 3,1|" # sorted by key, groups A then B -+ assert stable_out == "B: 3,1|A: 2,3,1|" # original order, groups B then A -+ -+ -+def test_groupby_stable_empty(env_async): -+ """Test stable groupby with empty input.""" -+ tmpl = env_async.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|length }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=[]) -+ assert out == "" -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"type": "A", "value": 1}, -+ {"value": 2}, # missing 'type' key -+ {"type": "A", "value": 3}, -+ ], -+) -+def test_groupby_stable_with_default(env_async, data): -+ """Test stable groupby with default parameter.""" -+ tmpl = env_async.from_string( -+ "{% for key, items in data()|groupby('type', default='X', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ assert out == "A: 1,3|X: 2|" -+ -+ -+@mark_dualiter( -+ "users", -+ lambda: [ -+ {"name": "emma", "city": "NY"}, -+ {"name": "john", "city": "NY"}, -+ {"name": "smith", "city": "WA"}, -+ ], -+) -+def test_groupby_with_counts(env_async, users): -+ tmpl = env_async.from_string( -+ "{% for city, items, count in users()|groupby('city', with_counts=true) %}" -+ "{{ city }}: {{ count }} users - {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ assert tmpl.render(users=users) == "NY: 2 users - emma, john\nWA: 1 users - smith\n" -+ -+ -+def test_groupby_with_counts_empty(env_async): -+ tmpl = env_async.from_string( -+ "{% for key, items, count in data|groupby('type', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ assert tmpl.render(data=[]) == "" -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"category": "books", "title": "Book1"}, -+ {"category": "books", "title": "Book2"}, -+ {"category": "movies", "title": "Movie1"}, -+ {"category": "games", "title": "Game1"}, -+ {"category": "games", "title": "Game2"}, -+ {"category": "games", "title": "Game3"}, -+ ], -+) -+def test_groupby_with_counts_multiple_groups(env_async, data): -+ tmpl = env_async.from_string( -+ "{% for key, items, count in data()|groupby('category', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ assert tmpl.render(data=data) == "books: 2\ngames: 3\nmovies: 1\n" -+ -+ -+@mark_dualiter( -+ "data", -+ lambda: [ -+ {"category": "books", "title": "Book1"}, -+ {"title": "Unknown1"}, -+ {"title": "Unknown2"}, -+ ], -+) -+def test_groupby_with_counts_and_default(env_async, data): -+ tmpl = env_async.from_string( -+ "{% for key, items, count in data()|groupby('category', default='misc', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ assert tmpl.render(data=data) == "books: 1\nmisc: 2\n" -+ -+ -+def make_multilevel_users(): -+ return [ -+ {"name": "alice", "department": "eng", "role": "dev"}, -+ {"name": "bob", "department": "eng", "role": "dev"}, -+ {"name": "charlie", "department": "eng", "role": "manager"}, -+ {"name": "diana", "department": "sales", "role": "rep"}, -+ {"name": "eve", "department": "sales", "role": "manager"}, -+ ] -+ -+ -+@mark_dualiter("users", make_multilevel_users) -+def test_groupby_multilevel_basic(env_async, users): -+ """Test basic multi-level grouping with 2 levels in async mode.""" -+ tmpl = env_async.from_string( -+ "{% for dept_group in users()|groupby(['department', 'role'], levels=2) %}" -+ "{{ dept_group.grouper }}:\n" -+ "{% for role_group in dept_group.list %}" -+ " {{ role_group.grouper }}: {{ role_group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ expected = ( -+ "eng:\n" -+ " dev: alice, bob\n" -+ " manager: charlie\n" -+ "sales:\n" -+ " manager: eve\n" -+ " rep: diana\n" -+ ) -+ assert out == expected -+ -+ -+def make_three_level_data(): -+ return [ -+ {"a": "x", "b": "1", "c": "i", "value": "A"}, -+ {"a": "x", "b": "1", "c": "ii", "value": "B"}, -+ {"a": "x", "b": "2", "c": "i", "value": "C"}, -+ {"a": "y", "b": "1", "c": "i", "value": "D"}, -+ ] -+ -+ -+@mark_dualiter("data", make_three_level_data) -+def test_groupby_multilevel_three_levels(env_async, data): -+ """Test multi-level grouping with 3 levels in async mode.""" -+ tmpl = env_async.from_string( -+ "{% for l1 in data()|groupby(['a', 'b', 'c'], levels=3) %}" -+ "{{ l1.grouper }}:\n" -+ "{% for l2 in l1.list %}" -+ " {{ l2.grouper }}:\n" -+ "{% for l3 in l2.list %}" -+ " {{ l3.grouper }}: {{ l3.list|map(attribute='value')|join(',') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ expected = ( -+ "x:\n" -+ " 1:\n" -+ " i: A\n" -+ " ii: B\n" -+ " 2:\n" -+ " i: C\n" -+ "y:\n" -+ " 1:\n" -+ " i: D\n" -+ ) -+ assert out == expected -+ -+ -+def test_groupby_multilevel_empty_data(env_async): -+ """Test multi-level grouping with empty data in async mode.""" -+ tmpl = env_async.from_string( -+ "{% for group in []|groupby(['a', 'b'], levels=2) %}" -+ "{{ group.grouper }}\n" -+ "{% else %}" -+ "No data\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render() -+ assert out == "No data\n" -+ -+ -+def test_groupby_multilevel_backward_compatibility(env_async): -+ """Test that single-level groupby still works in async mode.""" -+ tmpl = env_async.from_string( -+ "{% for group in data|groupby('category') %}" -+ "{{ group.grouper }}: {{ group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"name": "apple", "category": "fruit"}, -+ {"name": "banana", "category": "fruit"}, -+ {"name": "carrot", "category": "vegetable"}, -+ ] -+ out = tmpl.render(data=data) -+ expected = "fruit: apple, banana\nvegetable: carrot\n" -+ assert out == expected -+ -+ - @mark_dualiter("int_items", lambda: [1, 2, 3]) - def test_join_env_int(env_async, int_items): - tmpl = env_async.from_string('{{ items()|join("|") }}') -@@ -251,3 +860,214 @@ def test_custom_async_iteratable_filter(env_async, items): - ) - out = tmpl.render(items=items) - assert out == "0,1,2 .. 3,4,5" -+ -+ -+def make_users_for_filter(): -+ return [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ] -+ -+ -+@mark_dualiter("users", make_users_for_filter) -+def test_groupby_filter_groups_duplicates(env_async, users): -+ """Test filter_groups to find duplicates (groups with more than 1 item).""" -+ -+ def filter_duplicates(group): -+ return len(group) > 1 -+ -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, filter_func=filter_duplicates) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ -+@mark_dualiter("users", make_users_for_filter) -+def test_groupby_filter_groups_unique(env_async, users): -+ """Test filter_groups to find unique items (groups with exactly 1 item).""" -+ -+ def filter_unique(group): -+ return len(group) == 1 -+ -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, filter_func=filter_unique) -+ assert out == "LA: charlie\n" -+ -+ -+def make_users_for_empty_filter(): -+ return [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ] -+ -+ -+@mark_dualiter("users", make_users_for_empty_filter) -+def test_groupby_filter_groups_empty_result(env_async, users): -+ """Test filter_groups that filters out all groups.""" -+ -+ def filter_large_groups(group): -+ return len(group) > 10 -+ -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, filter_func=filter_large_groups) -+ assert out == "" -+ -+ -+@mark_dualiter("users", make_users_for_empty_filter) -+def test_groupby_filter_groups_all_pass(env_async, users): -+ """Test filter_groups that allows all groups to pass.""" -+ -+ def filter_all_pass(group): -+ return True -+ -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, filter_func=filter_all_pass) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ -+def make_users_with_missing_city(): -+ return [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie"}, -+ {"name": "david"}, -+ ] -+ -+ -+@mark_dualiter("users", make_users_with_missing_city) -+def test_groupby_filter_groups_with_default(env_async, users): -+ """Test filter_groups combined with default parameter.""" -+ -+ def filter_duplicates(group): -+ return len(group) > 1 -+ -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', default='Unknown', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, filter_func=filter_duplicates) -+ assert out == "NY: alice, bob\nUnknown: charlie, david\n" -+ -+ -+@mark_dualiter("users", make_users_for_empty_filter) -+def test_groupby_filter_groups_none(env_async, users): -+ """Test that filter_groups=None behaves like normal groupby.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=none) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ -+@mark_dualiter("users", make_users_for_filter) -+def test_groupby_filter_groups_lambda_duplicates(env_async, users): -+ """Test filter_groups with lambda function for duplicates in async environment.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, lambda_func=lambda group: len(group) > 1) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ -+@mark_dualiter("users", make_users_for_filter) -+def test_groupby_filter_groups_lambda_unique(env_async, users): -+ """Test filter_groups with lambda function for unique items in async environment.""" -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, lambda_func=lambda group: len(group) == 1) -+ assert out == "LA: charlie\n" -+ -+ -+def make_students_for_complex_filter(): -+ return [ -+ {"name": "alice", "score": 95}, -+ {"name": "bob", "score": 95}, -+ {"name": "charlie", "score": 75}, -+ {"name": "david", "score": 75}, -+ {"name": "eve", "score": 85}, -+ {"name": "frank", "score": 85}, -+ {"name": "grace", "score": 85}, -+ ] -+ -+ -+@mark_dualiter("students", make_students_for_complex_filter) -+def test_groupby_filter_groups_lambda_complex(env_async, students): -+ """Test filter_groups with complex lambda condition in async environment.""" -+ tmpl = env_async.from_string( -+ "{% for score, items in students()|groupby('score', filter_groups=lambda_func) %}" -+ "Score {{ score }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ students=students, -+ lambda_func=lambda group: len(group) >= 2 and group[0]["score"] >= 80, -+ ) -+ assert out == "Score 85: eve, frank, grace\nScore 95: alice, bob\n" -+ -+ -+@mark_dualiter("users", make_users_for_filter) -+def test_groupby_filter_groups_lambda_from_globals(env_async, users): -+ """Test filter_groups with lambda function from environment globals in async environment.""" -+ env_async.globals['only_dupes'] = lambda g: len(g) > 1 -+ env_async.globals['only_singles'] = lambda g: len(g) == 1 -+ -+ # Test duplicates from globals -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=only_dupes) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ # Test singles from globals -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=only_singles) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "LA: charlie\n" -+ -+ -+@mark_dualiter("users", make_users_for_empty_filter) -+def test_groupby_filter_groups_lambda_edge_cases(env_async, users): -+ """Test filter_groups lambda edge cases in async environment (filter all, allow all).""" -+ # Lambda that filters everything -+ tmpl = env_async.from_string( -+ "{% for city, items in users()|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, lambda_func=lambda group: False) -+ assert out == "" -+ -+ # Lambda that allows everything -+ out = tmpl.render(users=users, lambda_func=lambda group: True) -+ assert out == "LA: bob\nNY: alice\n" -diff --git a/tests/test_filters.py b/tests/test_filters.py -index 43ddf59c..bb28590d 100644 ---- a/tests/test_filters.py -+++ b/tests/test_filters.py -@@ -619,6 +619,1046 @@ class TestFilter: - ) - assert out == "NY: emma, john\nWA: smith\n" - -+ @pytest.mark.parametrize( -+ ("case_sensitive", "expect"), -+ [ -+ (False, "a: 1, 3\nb: 2\n"), -+ (True, "A: 3\na: 1\nb: 2\n"), -+ ], -+ ) -+ def test_groupby_case(self, env, case_sensitive, expect): -+ tmpl = env.from_string( -+ "{% for k, vs in data|groupby('k', case_sensitive=cs) %}" -+ "{{ k }}: {{ vs|join(', ', attribute='v') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[{"k": "a", "v": 1}, {"k": "b", "v": 2}, {"k": "A", "v": 3}], -+ cs=case_sensitive, -+ ) -+ assert out == expect -+ -+ def test_groupby_reverse(self, env): -+ tmpl = env.from_string( -+ """ -+ {%- for grouper, list in [{'foo': 1, 'bar': 2}, -+ {'foo': 2, 'bar': 3}, -+ {'foo': 1, 'bar': 1}, -+ {'foo': 3, 'bar': 4}]|groupby('foo', reverse=True) -%} -+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| -+ {%- endfor %}""" -+ ) -+ assert tmpl.render().split("|") == ["3: 3, 4", "2: 2, 3", "1: 1, 2: 1, 1", ""] -+ -+ def test_groupby_reverse_with_default(self, env): -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', default='NY', reverse=True) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "emma", "city": "NY"}, -+ {"name": "smith", "city": "WA"}, -+ {"name": "john"}, -+ ] -+ ) -+ assert out == "WA: smith\nNY: emma, john\n" -+ -+ def test_groupby_reverse_numeric(self, env): -+ tmpl = env.from_string( -+ "{% for score, items in data|groupby('score', reverse=True) %}" -+ "{{ score }}: {{ items|map(attribute='name')|join(', ') }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"name": "alice", "score": 95}, -+ {"name": "bob", "score": 87}, -+ {"name": "charlie", "score": 95}, -+ {"name": "david", "score": 72}, -+ ] -+ ) -+ assert out == "95: alice, charlie|87: bob|72: david|" -+ -+ def test_groupby_reverse_false(self, env): -+ # Test that reverse=False works the same as default behavior -+ tmpl = env.from_string( -+ """ -+ {%- for grouper, list in [{'foo': 1, 'bar': 2}, -+ {'foo': 2, 'bar': 3}, -+ {'foo': 1, 'bar': 1}, -+ {'foo': 3, 'bar': 4}]|groupby('foo', reverse=False) -%} -+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| -+ {%- endfor %}""" -+ ) -+ assert tmpl.render().split("|") == ["1: 1, 2: 1, 1", "2: 2, 3", "3: 3, 4", ""] -+ -+ def test_groupby_separator(self, env): -+ # Test with double underscore separator (Django-style) -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('profile__city', separator='__') %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "profile": {"city": "NYC"}}, -+ {"name": "bob", "profile": {"city": "LA"}}, -+ {"name": "charlie", "profile": {"city": "NYC"}}, -+ ] -+ ) -+ assert out == "LA: bob\nNYC: alice, charlie\n" -+ -+ def test_groupby_separator_with_default(self, env): -+ # Test separator with default value -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('profile__city', default='Unknown', separator='__') %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "profile": {"city": "NYC"}}, -+ {"name": "bob"}, # Missing profile -+ {"name": "charlie", "profile": {}}, # Missing city -+ ] -+ ) -+ assert out == "NYC: alice\nUnknown: bob, charlie\n" -+ -+ def test_groupby_custom_separator(self, env): -+ # Test with custom separator -+ tmpl = env.from_string( -+ "{% for value, items in data|groupby('a->b->c', separator='->') %}" -+ "{{ value }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"a": {"b": {"c": "x"}}}, -+ {"a": {"b": {"c": "y"}}}, -+ {"a": {"b": {"c": "x"}}}, -+ ] -+ ) -+ assert out == "x: 2\ny: 1\n" -+ -+ def test_groupby_max_groups(self, env): -+ """Test max_groups parameter limits the number of groups returned.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=2) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie", "city": "SF"}, -+ {"name": "david", "city": "NY"}, -+ {"name": "eve", "city": "LA"}, -+ ] -+ ) -+ # Should only return first 2 groups (LA and NY, alphabetically sorted) -+ assert out == "LA: bob, eve\nNY: alice, david\n" -+ -+ def test_groupby_max_groups_zero(self, env): -+ """Test max_groups=0 returns empty result.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=0) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ] -+ ) -+ assert out == "" -+ -+ def test_groupby_max_groups_larger_than_available(self, env): -+ """Test max_groups larger than available groups returns all groups.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=10) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ] -+ ) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ def test_groupby_max_groups_with_default(self, env): -+ """Test max_groups works with default parameter.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', default='Unknown', max_groups=2) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie"}, # No city, will use default -+ {"name": "david", "city": "SF"}, -+ ] -+ ) -+ # Should return first 2 groups alphabetically: LA, NY -+ assert out == "LA: bob\nNY: alice\n" -+ -+ def test_groupby_max_groups_empty_input(self, env): -+ """Test max_groups with empty input.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=2) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=[]) -+ assert out == "" -+ -+ def test_groupby_max_groups_none(self, env): -+ """Test max_groups=None returns all groups (default behavior).""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=none) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie", "city": "SF"}, -+ ] -+ ) -+ assert out == "LA: bob\nNY: alice\nSF: charlie\n" -+ -+ def test_groupby_max_groups_single_group(self, env): -+ """Test max_groups=1 returns only first group.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', max_groups=1) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ {"name": "charlie", "city": "SF"}, -+ {"name": "david", "city": "NY"}, -+ ] -+ ) -+ # Should return only LA (first alphabetically) with bob in that group -+ assert out == "LA: bob\n" -+ -+ def test_groupby_max_groups_with_tuple_index(self, env): -+ """Test max_groups works with tuple indexing.""" -+ tmpl = env.from_string( -+ "{% for grouper, items in data|groupby(0, max_groups=2) %}" -+ "{{ grouper }}: {{ items|map(attribute='1')|join(',') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ ("a", 1), -+ ("b", 2), -+ ("a", 3), -+ ("c", 4), -+ ("b", 5), -+ ] -+ ) -+ # Should return first 2 groups: a, b -+ assert out == "a: 1,3\nb: 2,5\n" -+ -+ def test_groupby_include_empty_true(self, env): -+ """Test groupby with include_empty=True (default behavior)""" -+ tmpl = env.from_string( -+ "{% for status, items in users|groupby('status', include_empty=true) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve"}, -+ ] -+ ) -+ # Should include empty string, None, and missing attribute groups -+ assert "active: alice, charlie\n" in out -+ assert ": bob\n" in out # empty string group -+ assert "None: dave\n" in out # None group -+ assert "None: eve\n" in out or ": eve\n" in out # missing attribute -+ -+ def test_groupby_include_empty_false(self, env): -+ """Test groupby with include_empty=False excludes falsy values""" -+ tmpl = env.from_string( -+ "{% for status, items in users|groupby('status', include_empty=false) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve", "status": 0}, -+ {"name": "frank"}, -+ ] -+ ) -+ # Should only include non-falsy values -+ assert out == "active: alice, charlie\n" -+ -+ def test_groupby_include_empty_with_default(self, env): -+ """Test groupby with include_empty=False and default value""" -+ tmpl = env.from_string( -+ "{% for status, items in users|groupby('status', default='unknown', include_empty=false) %}" -+ "{{ status }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": "active"}, -+ {"name": "dave", "status": None}, -+ {"name": "eve"}, -+ ] -+ ) -+ # Should include default value but exclude empty/None -+ assert "active: alice, charlie\n" in out -+ assert "unknown: eve\n" in out -+ # Should not include empty string or None groups -+ assert ": bob" not in out -+ assert "None: dave" not in out -+ -+ def test_groupby_include_empty_edge_cases(self, env): -+ """Test groupby include_empty with various falsy values""" -+ tmpl = env.from_string( -+ "{% for val, items in data|groupby('value', include_empty=false) %}" -+ "{{ val }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"value": "truthy"}, -+ {"value": ""}, -+ {"value": None}, -+ {"value": 0}, -+ {"value": []}, -+ {"value": {}}, -+ {"value": False}, -+ {"value": "another"}, -+ ] -+ ) -+ # Should only include truthy values -+ assert "truthy: 1\n" in out -+ assert "another: 1\n" in out -+ # Should not include any falsy values -+ assert ": 1" not in out.replace("truthy: 1", "").replace("another: 1", "") -+ assert "None:" not in out -+ assert "0:" not in out -+ assert "[]:" not in out -+ assert "{}:" not in out -+ assert "False:" not in out -+ -+ def test_groupby_include_empty_backward_compatibility(self, env): -+ """Test that default behavior (include_empty=True) maintains backward compatibility""" -+ # Test without include_empty parameter (should default to True) -+ tmpl1 = env.from_string( -+ "{% for status, items in users|groupby('status') %}" -+ "{{ status }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ # Test with explicit include_empty=True -+ tmpl2 = env.from_string( -+ "{% for status, items in users|groupby('status', include_empty=true) %}" -+ "{{ status }}: {{ items|length }}\n" -+ "{% endfor %}" -+ ) -+ -+ users = [ -+ {"name": "alice", "status": "active"}, -+ {"name": "bob", "status": ""}, -+ {"name": "charlie", "status": None}, -+ ] -+ -+ out1 = tmpl1.render(users=users) -+ out2 = tmpl2.render(users=users) -+ -+ # Both should produce identical output -+ assert out1 == out2 -+ assert "active: 1\n" in out1 -+ assert ": 1\n" in out1 # empty string -+ assert "None: 1\n" in out1 # None value -+ -+ def test_groupby_key_transform(self, env): -+ # Test basic key transformation -+ tmpl = env.from_string( -+ """ -+ {%- for grouper, list in [{'foo': 1, 'bar': 2}, -+ {'foo': 2, 'bar': 3}, -+ {'foo': 1, 'bar': 1}]|groupby('foo', key_transform=upper_func) -%} -+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}| -+ {%- endfor %}""" -+ ) -+ -+ def upper_func(x): -+ return str(x).upper() -+ -+ result = tmpl.render(upper_func=upper_func) -+ assert result.split("|") == ["1: 1, 2: 1, 1", "2: 2, 3", ""] -+ -+ def test_groupby_key_transform_date(self, env): -+ # Test date formatting transformation -+ from datetime import date -+ -+ events = [ -+ {"name": "event1", "date": date(2024, 1, 1)}, -+ {"name": "event2", "date": date(2024, 1, 1)}, -+ {"name": "event3", "date": date(2024, 2, 1)}, -+ ] -+ -+ tmpl = env.from_string( -+ """ -+ {%- for month, list in events|groupby('date', key_transform=format_month) -%} -+ {{ month }}: {{ list|length }} events| -+ {%- endfor %}""" -+ ) -+ -+ def format_month(dt): -+ return dt.strftime("%Y-%m") -+ -+ result = tmpl.render(events=events, format_month=format_month) -+ assert result.split("|") == ["2024-01: 2 events", "2024-02: 1 events", ""] -+ -+ def test_groupby_key_transform_none(self, env): -+ # Test that None key_transform works like normal groupby -+ tmpl = env.from_string( -+ """ -+ {%- for grouper, list in [{'foo': 1}, {'foo': 2}, {'foo': 1}]|groupby('foo', key_transform=none) -%} -+ {{ grouper }}:{{ list|length }}| -+ {%- endfor %}""" -+ ) -+ result = tmpl.render() -+ assert result.split("|") == ["1:2", "2:1", ""] -+ -+ def test_groupby_stable_basic(self, env): -+ """Test basic stable groupby functionality preserves order within groups.""" -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"type": "A", "value": 3}, -+ {"type": "B", "value": 1}, -+ {"type": "A", "value": 1}, -+ {"type": "B", "value": 2}, -+ {"type": "A", "value": 2}, -+ ] -+ out = tmpl.render(data=data) -+ assert out == "A: 3,1,2|B: 1,2|" -+ -+ def test_groupby_stable_vs_default(self, env): -+ """Test that stable=True preserves order while default behavior sorts by key.""" -+ data = [ -+ {"type": "B", "value": 3}, -+ {"type": "A", "value": 2}, -+ {"type": "B", "value": 1}, -+ {"type": "A", "value": 3}, -+ {"type": "A", "value": 1}, -+ ] -+ -+ # Default behavior (sorts by grouping key first) -+ tmpl_default = env.from_string( -+ "{% for key, items in data|groupby('type') %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ -+ # Stable behavior (preserves original order) -+ tmpl_stable = env.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ -+ default_out = tmpl_default.render(data=data) -+ stable_out = tmpl_stable.render(data=data) -+ -+ assert default_out == "A: 2,3,1|B: 3,1|" # sorted by key, groups A then B -+ assert stable_out == "B: 3,1|A: 2,3,1|" # original order, groups B then A -+ -+ def test_groupby_stable_empty(self, env): -+ """Test stable groupby with empty input.""" -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|length }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=[]) -+ assert out == "" -+ -+ def test_groupby_stable_single_group(self, env): -+ """Test stable groupby with single group.""" -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"type": "A", "value": 3}, -+ {"type": "A", "value": 1}, -+ {"type": "A", "value": 2}, -+ ] -+ out = tmpl.render(data=data) -+ assert out == "A: 3,1,2|" -+ -+ def test_groupby_stable_with_default(self, env): -+ """Test stable groupby with default parameter.""" -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('type', default='X', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"type": "A", "value": 1}, -+ {"value": 2}, # missing 'type' key -+ {"type": "A", "value": 3}, -+ ] -+ out = tmpl.render(data=data) -+ assert out == "A: 1,3|X: 2|" -+ -+ def test_groupby_stable_large_dataset(self, env): -+ """Test stable groupby with larger dataset to ensure performance.""" -+ data = [] -+ for i in range(100): -+ data.append({"type": chr(65 + (i % 3)), "value": i}) # A, B, C cycling -+ -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('type', stable=True) %}" -+ "{{ key }}: {{ items|length }}|" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=data) -+ # Should have groups A, B, C with roughly equal sizes -+ assert "A: 34|B: 33|C: 33|" == out -+ -+ def test_groupby_stable_nested_attribute(self, env): -+ """Test stable groupby with nested attribute access.""" -+ tmpl = env.from_string( -+ "{% for key, items in data|groupby('meta.category', stable=True) %}" -+ "{{ key }}: {{ items|map(attribute='value')|join(',') }}|" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"meta": {"category": "X"}, "value": 3}, -+ {"meta": {"category": "Y"}, "value": 1}, -+ {"meta": {"category": "X"}, "value": 2}, -+ ] -+ out = tmpl.render(data=data) -+ assert out == "X: 3,2|Y: 1|" -+ -+ def test_groupby_stable_boolean_false(self, env): -+ """Test that stable=False behaves like default (sorted).""" -+ data = [ -+ {"type": "A", "value": 3}, -+ {"type": "A", "value": 1}, -+ {"type": "A", "value": 2}, -+ ] -+ -+ tmpl_false = env.from_string( -+ "{% for key, items in data|groupby('type', stable=False) %}" -+ "{{ items|map(attribute='value')|join(',') }}" -+ "{% endfor %}" -+ ) -+ -+ tmpl_default = env.from_string( -+ "{% for key, items in data|groupby('type') %}" -+ "{{ items|map(attribute='value')|join(',') }}" -+ "{% endfor %}" -+ ) -+ -+ false_out = tmpl_false.render(data=data) -+ default_out = tmpl_default.render(data=data) -+ -+ assert ( -+ false_out == default_out == "3,1,2" -+ ) # Both preserve original order within groups -+ -+ def test_groupby_with_counts(self, env): -+ tmpl = env.from_string( -+ "{% for city, items, count in users|groupby('city', with_counts=true) %}" -+ "{{ city }}: {{ count }} users - {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "emma", "city": "NY"}, -+ {"name": "john", "city": "NY"}, -+ {"name": "smith", "city": "WA"}, -+ ] -+ ) -+ assert out == "NY: 2 users - emma, john\nWA: 1 users - smith\n" -+ -+ def test_groupby_with_counts_empty_groups(self, env): -+ tmpl = env.from_string( -+ "{% for key, items, count in data|groupby('type', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(data=[]) -+ assert out == "" -+ -+ def test_groupby_with_counts_single_group(self, env): -+ tmpl = env.from_string( -+ "{% for key, items, count in data|groupby('type', with_counts=true) %}" -+ "{{ key }}: {{ count }} items\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"type": "A", "value": 1}, -+ {"type": "A", "value": 2}, -+ {"type": "A", "value": 3}, -+ ] -+ ) -+ assert out == "A: 3 items\n" -+ -+ def test_groupby_with_counts_multiple_groups(self, env): -+ tmpl = env.from_string( -+ "{% for key, items, count in data|groupby('category', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"category": "books", "title": "Book1"}, -+ {"category": "books", "title": "Book2"}, -+ {"category": "movies", "title": "Movie1"}, -+ {"category": "games", "title": "Game1"}, -+ {"category": "games", "title": "Game2"}, -+ {"category": "games", "title": "Game3"}, -+ ] -+ ) -+ assert out == "books: 2\ngames: 3\nmovies: 1\n" -+ -+ def test_groupby_with_counts_and_default(self, env): -+ tmpl = env.from_string( -+ "{% for key, items, count in data|groupby('category', default='misc', with_counts=true) %}" -+ "{{ key }}: {{ count }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ data=[ -+ {"category": "books", "title": "Book1"}, -+ {"title": "Unknown1"}, -+ {"title": "Unknown2"}, -+ ] -+ ) -+ assert out == "books: 1\nmisc: 2\n" -+ -+ def test_groupby_multilevel_basic(self, env): -+ """Test basic multi-level grouping with 2 levels.""" -+ tmpl = env.from_string( -+ "{% for dept_group in users|groupby(['department', 'role'], levels=2) %}" -+ "{{ dept_group.grouper }}:\n" -+ "{% for role_group in dept_group.list %}" -+ " {{ role_group.grouper }}: {{ role_group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ users = [ -+ {"name": "alice", "department": "eng", "role": "dev"}, -+ {"name": "bob", "department": "eng", "role": "dev"}, -+ {"name": "charlie", "department": "eng", "role": "manager"}, -+ {"name": "diana", "department": "sales", "role": "rep"}, -+ {"name": "eve", "department": "sales", "role": "manager"}, -+ ] -+ out = tmpl.render(users=users) -+ expected = "eng:\n dev: alice, bob\n manager: charlie\nsales:\n manager: eve\n rep: diana\n" -+ assert out == expected -+ -+ def test_groupby_multilevel_three_levels(self, env): -+ """Test multi-level grouping with 3 levels.""" -+ tmpl = env.from_string( -+ "{% for l1 in data|groupby(['a', 'b', 'c'], levels=3) %}" -+ "{{ l1.grouper }}:\n" -+ "{% for l2 in l1.list %}" -+ " {{ l2.grouper }}:\n" -+ "{% for l3 in l2.list %}" -+ " {{ l3.grouper }}: {{ l3.list|map(attribute='value')|join(',') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"a": "x", "b": "1", "c": "i", "value": "A"}, -+ {"a": "x", "b": "1", "c": "ii", "value": "B"}, -+ {"a": "x", "b": "2", "c": "i", "value": "C"}, -+ {"a": "y", "b": "1", "c": "i", "value": "D"}, -+ ] -+ out = tmpl.render(data=data) -+ expected = "x:\n 1:\n i: A\n ii: B\n 2:\n i: C\ny:\n 1:\n i: D\n" -+ assert out == expected -+ -+ def test_groupby_multilevel_with_default(self, env): -+ """Test multi-level grouping with default values.""" -+ tmpl = env.from_string( -+ "{% for dept_group in users|groupby(['department', 'role'], levels=2, default='unknown') %}" -+ "{{ dept_group.grouper }}:\n" -+ "{% for role_group in dept_group.list %}" -+ " {{ role_group.grouper }}: {{ role_group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ users = [ -+ {"name": "alice", "department": "eng", "role": "dev"}, -+ {"name": "bob", "department": "eng"}, # missing role -+ {"name": "charlie", "role": "manager"}, # missing department -+ {"name": "diana"}, # missing both -+ ] -+ out = tmpl.render(users=users) -+ expected = "eng:\n dev: alice\n unknown: bob\nunknown:\n manager: charlie\n unknown: diana\n" -+ assert out == expected -+ -+ def test_groupby_multilevel_empty_data(self, env): -+ """Test multi-level grouping with empty data.""" -+ tmpl = env.from_string( -+ "{% for group in data|groupby(['a', 'b'], levels=2) %}{{ group.grouper }}\n{% else %}No data\n{% endfor %}" -+ ) -+ out = tmpl.render(data=[]) -+ assert out == "No data\n" -+ -+ def test_groupby_multilevel_single_item(self, env): -+ """Test multi-level grouping with single item.""" -+ tmpl = env.from_string( -+ "{% for l1 in data|groupby(['a', 'b'], levels=2) %}" -+ "{{ l1.grouper }}:\n" -+ "{% for l2 in l1.list %}" -+ " {{ l2.grouper }}: {{ l2.list|map(attribute='value')|join(',') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ data = [{"a": "x", "b": "y", "value": "test"}] -+ out = tmpl.render(data=data) -+ expected = "x:\n y: test\n" -+ assert out == expected -+ -+ def test_groupby_multilevel_dot_notation(self, env): -+ """Test multi-level grouping with dot notation attributes.""" -+ tmpl = env.from_string( -+ "{% for l1 in data|groupby(['obj.a', 'obj.b'], levels=2) %}" -+ "{{ l1.grouper }}:\n" -+ "{% for l2 in l1.list %}" -+ " {{ l2.grouper }}: {{ l2.list|map(attribute='value')|join(',') }}\n" -+ "{% endfor %}" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"obj": {"a": "x", "b": "1"}, "value": "A"}, -+ {"obj": {"a": "x", "b": "2"}, "value": "B"}, -+ {"obj": {"a": "y", "b": "1"}, "value": "C"}, -+ ] -+ out = tmpl.render(data=data) -+ expected = "x:\n 1: A\n 2: B\ny:\n 1: C\n" -+ assert out == expected -+ -+ def test_groupby_backward_compatibility(self, env): -+ """Test that single-level groupby still works as before.""" -+ tmpl = env.from_string( -+ "{% for group in data|groupby('category') %}" -+ "{{ group.grouper }}: {{ group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"name": "apple", "category": "fruit"}, -+ {"name": "banana", "category": "fruit"}, -+ {"name": "carrot", "category": "vegetable"}, -+ ] -+ out = tmpl.render(data=data) -+ expected = "fruit: apple, banana\nvegetable: carrot\n" -+ assert out == expected -+ -+ def test_groupby_single_attribute_list(self, env): -+ """Test that single attribute in list works without levels.""" -+ tmpl = env.from_string( -+ "{% for group in data|groupby(['category']) %}" -+ "{{ group.grouper }}: {{ group.list|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ data = [ -+ {"name": "apple", "category": "fruit"}, -+ {"name": "banana", "category": "fruit"}, -+ ] -+ out = tmpl.render(data=data) -+ expected = "fruit: apple, banana\n" -+ assert out == expected -+ -+ def test_groupby_filter_groups_duplicates(self, env): -+ """Test filter_groups to find duplicates (groups with more than 1 item).""" -+ -+ def filter_duplicates(group): -+ return len(group) > 1 -+ -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ], -+ filter_func=filter_duplicates, -+ ) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ def test_groupby_filter_groups_unique(self, env): -+ """Test filter_groups to find unique items (groups with exactly 1 item).""" -+ -+ def filter_unique(group): -+ return len(group) == 1 -+ -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ], -+ filter_func=filter_unique, -+ ) -+ assert out == "LA: charlie\n" -+ -+ def test_groupby_filter_groups_empty_result(self, env): -+ """Test filter_groups that filters out all groups.""" -+ -+ def filter_large_groups(group): -+ return len(group) > 10 -+ -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ], -+ filter_func=filter_large_groups, -+ ) -+ assert out == "" -+ -+ def test_groupby_filter_groups_all_pass(self, env): -+ """Test filter_groups that allows all groups to pass.""" -+ -+ def filter_all_pass(group): -+ return True -+ -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ], -+ filter_func=filter_all_pass, -+ ) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ def test_groupby_filter_groups_with_default(self, env): -+ """Test filter_groups combined with default parameter.""" -+ -+ def filter_duplicates(group): -+ return len(group) > 1 -+ -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', default='Unknown', filter_groups=filter_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie"}, -+ {"name": "david"}, -+ ], -+ filter_func=filter_duplicates, -+ ) -+ assert out == "NY: alice, bob\nUnknown: charlie, david\n" -+ -+ def test_groupby_filter_groups_complex_condition(self, env): -+ """Test filter_groups with complex filtering condition.""" -+ -+ def filter_high_scoring_groups(group): -+ return len(group) >= 2 and group[0]["score"] >= 80 -+ -+ tmpl = env.from_string( -+ "{% for score, items in students|groupby('score', filter_groups=filter_func) %}" -+ "Score {{ score }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ students=[ -+ {"name": "alice", "score": 95}, -+ {"name": "bob", "score": 95}, -+ {"name": "charlie", "score": 75}, -+ {"name": "david", "score": 75}, -+ {"name": "eve", "score": 85}, -+ {"name": "frank", "score": 85}, -+ {"name": "grace", "score": 85}, -+ ], -+ filter_func=filter_high_scoring_groups, -+ ) -+ assert out == "Score 85: eve, frank, grace\nScore 95: alice, bob\n" -+ -+ def test_groupby_filter_groups_none(self, env): -+ """Test that filter_groups=None behaves like normal groupby.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=none) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ] -+ ) -+ assert out == "LA: bob\nNY: alice\n" -+ -+ def test_groupby_filter_groups_lambda_duplicates(self, env): -+ """Test filter_groups with lambda function for duplicates.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ], -+ lambda_func=lambda group: len(group) > 1, -+ ) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ def test_groupby_filter_groups_lambda_unique(self, env): -+ """Test filter_groups with lambda function for unique items.""" -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ users=[ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ], -+ lambda_func=lambda group: len(group) == 1, -+ ) -+ assert out == "LA: charlie\n" -+ -+ def test_groupby_filter_groups_lambda_complex(self, env): -+ """Test filter_groups with complex lambda condition.""" -+ tmpl = env.from_string( -+ "{% for score, items in students|groupby('score', filter_groups=lambda_func) %}" -+ "Score {{ score }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render( -+ students=[ -+ {"name": "alice", "score": 95}, -+ {"name": "bob", "score": 95}, -+ {"name": "charlie", "score": 75}, -+ {"name": "david", "score": 75}, -+ {"name": "eve", "score": 85}, -+ {"name": "frank", "score": 85}, -+ {"name": "grace", "score": 85}, -+ ], -+ lambda_func=lambda group: len(group) >= 2 and group[0]["score"] >= 80, -+ ) -+ assert out == "Score 85: eve, frank, grace\nScore 95: alice, bob\n" -+ -+ def test_groupby_filter_groups_lambda_from_globals(self, env): -+ """Test filter_groups with lambda function from environment globals.""" -+ env.globals['only_dupes'] = lambda g: len(g) > 1 -+ env.globals['only_singles'] = lambda g: len(g) == 1 -+ -+ users = [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "NY"}, -+ {"name": "charlie", "city": "LA"}, -+ {"name": "david", "city": "SF"}, -+ {"name": "eve", "city": "SF"}, -+ ] -+ -+ # Test duplicates from globals -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=only_dupes) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "NY: alice, bob\nSF: david, eve\n" -+ -+ # Test singles from globals -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=only_singles) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users) -+ assert out == "LA: charlie\n" -+ -+ def test_groupby_filter_groups_lambda_edge_cases(self, env): -+ """Test filter_groups lambda edge cases (filter all, allow all).""" -+ users = [ -+ {"name": "alice", "city": "NY"}, -+ {"name": "bob", "city": "LA"}, -+ ] -+ -+ # Lambda that filters everything -+ tmpl = env.from_string( -+ "{% for city, items in users|groupby('city', filter_groups=lambda_func) %}" -+ "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n" -+ "{% endfor %}" -+ ) -+ out = tmpl.render(users=users, lambda_func=lambda group: False) -+ assert out == "" -+ -+ # Lambda that allows everything -+ out = tmpl.render(users=users, lambda_func=lambda group: True) -+ assert out == "LA: bob\nNY: alice\n" -+ - def test_filtertag(self, env): - tmpl = env.from_string( - "{% filter upper|replace('FOO', 'foo') %}foobar{% endfilter %}" diff --git a/dataset/pallets_jinja_task/task1559/combined.patch b/dataset/pallets_jinja_task/task1559/combined.patch index a908ed9d..b34ec5a8 100644 --- a/dataset/pallets_jinja_task/task1559/combined.patch +++ b/dataset/pallets_jinja_task/task1559/combined.patch @@ -1,5 +1,5 @@ diff --git a/src/jinja2/ext.py b/src/jinja2/ext.py -index d21b83aa..3339338f 100644 +index d21b83aa..305730c9 100644 --- a/src/jinja2/ext.py +++ b/src/jinja2/ext.py @@ -181,6 +181,29 @@ def _make_new_gettext(func: t.Callable[[str], str]) -> t.Callable[..., str]: @@ -210,7 +210,7 @@ index d21b83aa..3339338f 100644 self.environment.globals.pop(key, None) def _extract( -@@ -351,20 +513,76 @@ class InternationalizationExtension(Extension): +@@ -351,20 +513,77 @@ class InternationalizationExtension(Extension): source = self.environment.parse(source) return extract_from_ast(source, gettext_functions) @@ -279,6 +279,7 @@ index d21b83aa..3339338f 100644 plural_expr_assignment: t.Optional[nodes.Assign] = None + num_called_num = False variables: t.Dict[str, nodes.Expr] = {} ++ priority_expr: t.Optional[nodes.Expr] = None trimmed = None + domain: t.Optional[str] = None + metadata: t.Dict[str, str] = {} @@ -289,7 +290,7 @@ index d21b83aa..3339338f 100644 parser.stream.expect("comma") # skip colon for python compatibility -@@ -372,6 +590,54 @@ class InternationalizationExtension(Extension): +@@ -372,6 +591,54 @@ class InternationalizationExtension(Extension): break token = parser.stream.expect("name") @@ -344,10 +345,18 @@ index d21b83aa..3339338f 100644 if token.value in variables: parser.fail( f"translatable variable {token.value!r} defined twice.", -@@ -383,11 +649,16 @@ class InternationalizationExtension(Extension): +@@ -382,12 +649,23 @@ class InternationalizationExtension(Extension): + # expressions if parser.stream.current.type == "assign": next(parser.stream) - variables[token.value] = var = parser.parse_expression() +- variables[token.value] = var = parser.parse_expression() ++ var = parser.parse_expression() ++ # Handle priority parameter specially - don't add to variables ++ if token.value == "priority": ++ priority_expr = var ++ processed_any_param = True ++ continue ++ variables[token.value] = var + processed_any_param = True elif trimmed is None and token.value in ("trimmed", "notrimmed"): trimmed = token.value == "trimmed" @@ -361,7 +370,7 @@ index d21b83aa..3339338f 100644 if plural_expr is None: if isinstance(var, nodes.Call): -@@ -440,6 +711,9 @@ class InternationalizationExtension(Extension): +@@ -440,6 +718,9 @@ class InternationalizationExtension(Extension): if name not in variables: variables[name] = nodes.Name(name, "load") @@ -371,7 +380,7 @@ index d21b83aa..3339338f 100644 if not have_plural: plural_expr = None elif plural_expr is None: -@@ -455,10 +729,13 @@ class InternationalizationExtension(Extension): +@@ -455,10 +736,14 @@ class InternationalizationExtension(Extension): node = self._make_node( singular, plural, @@ -382,10 +391,11 @@ index d21b83aa..3339338f 100644 num_called_num and have_plural, + domain, + metadata, ++ priority_expr, ) node.set_lineno(lineno) if plural_expr_assignment is not None: -@@ -510,10 +787,13 @@ class InternationalizationExtension(Extension): +@@ -510,10 +795,14 @@ class InternationalizationExtension(Extension): self, singular: str, plural: t.Optional[str], @@ -396,27 +406,41 @@ index d21b83aa..3339338f 100644 num_called_num: bool, + domain: t.Optional[str] = None, + metadata: t.Dict[str, str] = None, ++ priority_expr: t.Optional[nodes.Expr] = None, ) -> nodes.Output: """Generates a useful node from the data provided.""" newstyle = self.environment.newstyle_gettext # type: ignore -@@ -528,19 +808,47 @@ class InternationalizationExtension(Extension): +@@ -526,49 +815,197 @@ class InternationalizationExtension(Extension): + if plural: + plural = plural.replace("%%", "%") - # singular only: - if plural_expr is None: +- # singular only: +- if plural_expr is None: - gettext = nodes.Name("gettext", "load") - node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) -+ if domain is not None: -+ # Use dgettext for domain-specific translation -+ dgettext = nodes.Name("dgettext", "load") -+ node = nodes.Call(dgettext, [nodes.Const(domain), nodes.Const(singular)], [], None, None) -+ elif context: -+ pgettext = nodes.Name("pgettext", "load") -+ node = nodes.Call(pgettext, [nodes.Const(context), nodes.Const(singular)], [], None, None) ++ # If priority is specified, use priority-aware translation ++ if priority_expr is not None: ++ # Use extension method to handle priority-based translation ++ if plural_expr is None: ++ # singular only with priority ++ node = self.call_method( ++ "_priority_gettext", ++ [nodes.Const(singular), priority_expr], ++ ) + else: -+ gettext = nodes.Name("gettext", "load") -+ node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) ++ # plural with priority ++ node = self.call_method( ++ "_priority_ngettext", ++ [nodes.Const(singular), nodes.Const(plural), plural_expr, priority_expr], ++ ) - # singular and plural +- # singular and plural ++ # For priority-based translations, handle variables ++ if variables: ++ for key, value in variables.items(): ++ if num_called_num and key == "num": ++ continue ++ node.kwargs.append(nodes.Keyword(key, value)) else: - ngettext = nodes.Name("ngettext", "load") - node = nodes.Call( @@ -426,51 +450,203 @@ index d21b83aa..3339338f 100644 - None, - None, - ) -+ if domain is not None: -+ # Use dngettext for domain-specific plural translation -+ dngettext = nodes.Name("dngettext", "load") -+ node = nodes.Call( -+ dngettext, -+ [nodes.Const(domain), nodes.Const(singular), nodes.Const(plural), plural_expr], -+ [], -+ None, -+ None, -+ ) -+ elif context: -+ npgettext = nodes.Name("npgettext", "load") -+ node = nodes.Call( -+ npgettext, -+ [nodes.Const(context), nodes.Const(singular), nodes.Const(plural), plural_expr], -+ [], -+ None, -+ None, -+ ) ++ # Original behavior without priority ++ # singular only: ++ if plural_expr is None: ++ if domain is not None: ++ # Use dgettext for domain-specific translation ++ dgettext = nodes.Name("dgettext", "load") ++ node = nodes.Call(dgettext, [nodes.Const(domain), nodes.Const(singular)], [], None, None) ++ elif context: ++ pgettext = nodes.Name("pgettext", "load") ++ node = nodes.Call(pgettext, [nodes.Const(context), nodes.Const(singular)], [], None, None) ++ else: ++ gettext = nodes.Name("gettext", "load") ++ node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) + +- # in case newstyle gettext is used, the method is powerful +- # enough to handle the variable expansion and autoescape +- # handling itself +- if newstyle: +- for key, value in variables.items(): +- # the function adds that later anyways in case num was +- # called num, so just skip it. +- if num_called_num and key == "num": +- continue +- node.kwargs.append(nodes.Keyword(key, value)) ++ # singular and plural + else: -+ ngettext = nodes.Name("ngettext", "load") -+ node = nodes.Call( -+ ngettext, -+ [nodes.Const(singular), nodes.Const(plural), plural_expr], -+ [], -+ None, -+ None, -+ ) ++ if domain is not None: ++ # Use dngettext for domain-specific plural translation ++ dngettext = nodes.Name("dngettext", "load") ++ node = nodes.Call( ++ dngettext, ++ [nodes.Const(domain), nodes.Const(singular), nodes.Const(plural), plural_expr], ++ [], ++ None, ++ None, ++ ) ++ elif context: ++ npgettext = nodes.Name("npgettext", "load") ++ node = nodes.Call( ++ npgettext, ++ [nodes.Const(context), nodes.Const(singular), nodes.Const(plural), plural_expr], ++ [], ++ None, ++ None, ++ ) ++ else: ++ ngettext = nodes.Name("ngettext", "load") ++ node = nodes.Call( ++ ngettext, ++ [nodes.Const(singular), nodes.Const(plural), plural_expr], ++ [], ++ None, ++ None, ++ ) - # in case newstyle gettext is used, the method is powerful - # enough to handle the variable expansion and autoescape -@@ -568,7 +876,11 @@ class InternationalizationExtension(Extension): - ] - ), - ) -- return nodes.Output([node]) +- # otherwise do that here +- else: +- # mark the return value as safe if we are in an +- # environment with autoescaping turned on +- node = nodes.MarkSafeIfAutoescape(node) ++ # in case newstyle gettext is used, the method is powerful ++ # enough to handle the variable expansion and autoescape ++ # handling itself ++ if newstyle: ++ for key, value in variables.items(): ++ # the function adds that later anyways in case num was ++ # called num, so just skip it. ++ if num_called_num and key == "num": ++ continue ++ node.kwargs.append(nodes.Keyword(key, value)) ++ ++ # otherwise do that here ++ else: ++ # mark the return value as safe if we are in an ++ # environment with autoescaping turned on ++ node = nodes.MarkSafeIfAutoescape(node) ++ if variables: ++ node = nodes.Mod( ++ node, ++ nodes.Dict( ++ [ ++ nodes.Pair(nodes.Const(key), value) ++ for key, value in variables.items() ++ ] ++ ), ++ ) + # Store metadata as node attributes for translation tools to access + output_node = nodes.Output([node]) + if metadata: + output_node.trans_metadata = metadata # type: ignore + return output_node ++ ++ @pass_context ++ def _priority_gettext( ++ self, context: Context, message: str, priority: int, **variables: t.Any ++ ) -> str: ++ """Handle priority-aware gettext translation with fallback.""" ++ # Ensure priority is a valid integer ++ try: ++ priority = int(priority) ++ if priority < 0: ++ priority = 0 ++ except (ValueError, TypeError): ++ priority = 0 ++ ++ gettext_func = context.resolve("gettext") ++ if not gettext_func: ++ # No gettext function available, return original message ++ result = message ++ if context.eval_ctx.autoescape: ++ result = Markup(result) + if variables: +- node = nodes.Mod( +- node, +- nodes.Dict( +- [ +- nodes.Pair(nodes.Const(key), value) +- for key, value in variables.items() +- ] +- ), +- ) +- return nodes.Output([node]) ++ return result % variables # type: ignore ++ return result ++ ++ # Limit priority to reasonable maximum to prevent excessive calls ++ max_priority = min(priority, 10) ++ ++ # Try translation starting from priority 0 up to the specified priority ++ current_priority = 0 ++ ++ while current_priority <= max_priority: ++ # Try to get translation at current priority level ++ result = context.call(gettext_func, message) ++ ++ # Check if we got a translation (result != original message) ++ if result != message: ++ # Translation found! Apply formatting and return ++ if context.eval_ctx.autoescape: ++ result = Markup(result) ++ if variables: ++ return result % variables # type: ignore ++ return result ++ ++ # No translation found at this priority level ++ if current_priority >= max_priority: ++ break ++ ++ current_priority += 1 ++ ++ # Fallback to original message if no translation found ++ result = message ++ if context.eval_ctx.autoescape: ++ result = Markup(result) ++ if variables: ++ return result % variables # type: ignore ++ return result ++ ++ @pass_context ++ def _priority_ngettext( ++ self, ++ context: Context, ++ singular: str, ++ plural: str, ++ num: int, ++ priority: int, ++ **variables: t.Any, ++ ) -> str: ++ """Handle priority-aware ngettext translation with fallback.""" ++ # Ensure priority is a valid integer ++ try: ++ priority = int(priority) ++ if priority < 0: ++ priority = 0 ++ except (ValueError, TypeError): ++ priority = 0 ++ ++ variables.setdefault("num", num) ++ ++ ngettext_func = context.resolve("ngettext") ++ if not ngettext_func: ++ # No ngettext function available, return original message ++ result = singular if num == 1 else plural ++ if context.eval_ctx.autoescape: ++ result = Markup(result) ++ return result % variables # type: ignore ++ ++ # Call ngettext directly ++ result = context.call(ngettext_func, singular, plural, num) ++ ++ if context.eval_ctx.autoescape: ++ result = Markup(result) ++ return result % variables # type: ignore class ExprStmtExtension(Extension): -@@ -849,6 +1161,302 @@ def babel_extract( +@@ -849,6 +1286,302 @@ def babel_extract( #: nicer import names i18n = InternationalizationExtension diff --git a/dataset/react_hook_form_task/task85/Dockerfile b/dataset/react_hook_form_task/task85/Dockerfile index 60e2a23e..7772dfd3 100644 --- a/dataset/react_hook_form_task/task85/Dockerfile +++ b/dataset/react_hook_form_task/task85/Dockerfile @@ -10,7 +10,10 @@ RUN apt-get update && apt-get install -y \ && rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED # Install pnpm globally -RUN npm install -g pnpm +# Pinned: unpinned pnpm resolved to v10 in Sept 2026, which fails `pnpm install` with +# ERR_PNPM_IGNORED_BUILDS (it refuses to run @swc/core / cypress / msw build scripts without an +# explicit approve-builds step). The published amd64 image was built with pnpm 8; task153 pins 8 too. +RUN npm install -g pnpm@8 # Clone the repository and checkout the specific commit WORKDIR /workspace diff --git a/scripts/check_combined.py b/scripts/check_combined.py new file mode 100644 index 00000000..73d1f2f7 --- /dev/null +++ b/scripts/check_combined.py @@ -0,0 +1,138 @@ +"""Check every task's combined.patch passes every one of its features' tests. + +Third leg of the objective dataset check (the other two are in check_gradeable.py): + + tests alone -> must FAIL + tests + feature.patch -> must PASS + tests + combined.patch -> must PASS <- this script + +combined.patch is the full PR, i.e. what the "all features in one tree" reference looks like. +If a feature's tests fail against it, either the combined patch does not contain that feature +or its tests are incompatible with a sibling feature landing in the same tree — both mean the +merged-eval path can never score that pair. + +Same sandbox path as check_gradeable.py: the image's own `runner.sh`, one sandbox per feature. +`--backend modal` is linux/amd64; `--backend docker` is the local daemon (arm64 on Apple +Silicon), and the published images are multi-arch, so running both covers both architectures. + + python scripts/check_combined.py # all 199 on modal + python scripts/check_combined.py --backend docker # all 199 locally + python scripts/check_combined.py pillow_task/task290 # one task +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from check_gradeable import _amd64_ref # noqa: E402 + +from cooperbench.eval.backends import get_backend # noqa: E402 +from cooperbench.utils import get_image_name # noqa: E402 + +DATASET = Path(__file__).resolve().parents[1] / "dataset" + + +def report_path(backend: str) -> Path: + return DATASET / f"combined_report{'' if backend == 'modal' else '_' + backend}.json" + + +def _write(sb, path: str, content: str) -> None: + # Chunked: the whole command line rides in the exec's argv, and Modal caps that at 64 KiB + # (ARG_MAX). pallets_jinja/1465's combined.patch alone is >100 KB. + enc = base64.b64encode(content.encode()).decode() + sb.exec("bash", "-c", f": > {path}") + for i in range(0, len(enc), 32_000): + sb.exec("bash", "-c", f"echo '{enc[i : i + 32_000]}' | base64 -d >> {path}") + + +def check_feature(repo: str, task: str, feature: str, backend: str) -> dict: + td = DATASET / repo / task + tests, combined = td / feature / "tests.patch", td / "combined.patch" + out = {"task": f"{repo}/{task}", "feature": feature} + if not (tests.is_file() and combined.is_file()): + return {**out, "verdict": "SKIP", "note": "missing tests.patch or combined.patch"} + + image = get_image_name(repo, int(task.replace("task", ""))) + if backend == "modal": + image = _amd64_ref(image) + started = time.time() + sb = get_backend(backend).create_sandbox(image, timeout=3600) + try: + sb.exec("bash", "-c", "mkdir -p /patches") + _write(sb, "/patches/tests.patch", tests.read_text()) + _write(sb, "/patches/combined.patch", combined.read_text()) + # Same shape as check_gradeable.run: keep the runner's own exit status (never pipe it), + # then grep the signal lines back out of the log. + r = sb.exec( + "bash", + "-c", + "bash /usr/local/bin/runner.sh tests.patch combined.patch > /tmp/out.log 2>&1; echo RC=$?; " + "grep -iE 'does not apply|failed to apply|error:|FAILED|assert|" + "[0-9]+ (passed|failed)|no tests|collected|panic|cannot|Tests:|test result:|^--- FAIL' " + "/tmp/out.log | grep -viE 'Removing |Repository (cleaned|restored)' | tail -25", + ) + body = r.stdout_read() + r.stderr_read() + rc = next((int(ln[3:]) for ln in body.splitlines() if ln.startswith("RC=")), -1) + return { + **out, + "verdict": "OK" if rc == 0 else "COMBINED_FAILS", + "rc": rc, + "seconds": round(time.time() - started), + "tail": body.strip()[-600:], + } + except Exception as exc: + return {**out, "verdict": "ERROR", "note": f"{type(exc).__name__}: {exc}"} + finally: + sb.terminate() + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("task", nargs="?", help="repo/taskN, default all") + ap.add_argument("--workers", type=int, default=12) + ap.add_argument("--backend", choices=["modal", "docker"], default="modal") + args = ap.parse_args() + + feats = [ + (p.parent.parent.name, p.parent.name, p.name) + for p in sorted(DATASET.glob("*/task*/feature*")) + if p.is_dir() and (not args.task or f"{p.parent.parent.name}/{p.parent.name}" == args.task) + ] + if not feats: + raise SystemExit(f"no features matched {args.task!r}") + report = report_path(args.backend) + print(f"checking {len(feats)} features on {args.backend}, {args.workers} sandboxes at a time\n", flush=True) + + results, done = [], 0 + with ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = {pool.submit(check_feature, *f, args.backend): f for f in feats} + for fut in as_completed(futures): + r = fut.result() + results.append(r) + done += 1 + if r["verdict"] != "OK": + print( + f" [{done}/{len(feats)}] {r['verdict']:15s} {r['task']} {r['feature']} {r.get('note', '')}", + flush=True, + ) + elif done % 20 == 0: + print(f" [{done}/{len(feats)}] ...", flush=True) + + report.write_text(json.dumps(sorted(results, key=lambda r: (r["task"], r["feature"])), indent=1)) + tally: dict[str, int] = {} + for r in results: + tally[r["verdict"]] = tally.get(r["verdict"], 0) + 1 + print("\n" + " · ".join(f"{k} {v}" for k, v in sorted(tally.items()))) + print(f"report -> {report}") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_gradeable.py b/scripts/check_gradeable.py index 67b75bec..cb10623a 100644 --- a/scripts/check_gradeable.py +++ b/scripts/check_gradeable.py @@ -19,12 +19,14 @@ python scripts/check_gradeable.py # all 199 python scripts/check_gradeable.py pillow_task/task290 # one task python scripts/check_gradeable.py --workers 16 + python scripts/check_gradeable.py --backend docker # local daemon, i.e. arm64 on a Mac """ from __future__ import annotations import argparse import base64 +import functools import json import subprocess import sys @@ -38,10 +40,15 @@ from cooperbench.utils import get_image_name # noqa: E402 DATASET = Path(__file__).resolve().parents[1] / "dataset" -REPORT = DATASET / "gradeable_report.json" -REPORT_LOCAL = DATASET / "gradeable_report_localrunner.json" +def report_path(backend: str, local_runner: bool) -> Path: + suffix = "" if backend == "modal" else f"_{backend}" + suffix += "_localrunner" if local_runner else "" + return DATASET / f"gradeable_report{suffix}.json" + + +@functools.cache def _amd64_ref(image: str) -> str: """Pin the amd64 manifest by digest. @@ -49,10 +56,15 @@ def _amd64_ref(image: str) -> str: pick the arm64 manifest and fail the image build with "image architecture arm64 not supported" — then CACHE that failure, so every later run of that task fails instantly. Handing it a digest removes the choice. Falls back to the plain tag if the registry cannot be reached. + + Cached per image: each call is a manifest request against Docker Hub from THIS machine, which + anonymous pulls cap at 100/hour per IP — one call per feature (199) exhausted it mid-sweep and + starved every local `docker pull` for the next hour. """ try: - raw = subprocess.run(["docker", "buildx", "imagetools", "inspect", image, "--raw"], - capture_output=True, text=True, timeout=60) + raw = subprocess.run( + ["docker", "buildx", "imagetools", "inspect", image, "--raw"], capture_output=True, text=True, timeout=60 + ) if raw.returncode != 0: return image for m in json.loads(raw.stdout).get("manifests", []): @@ -65,20 +77,26 @@ def _amd64_ref(image: str) -> str: def _write(sb, path: str, content: str) -> None: + # Chunked: the whole command line rides in the exec's argv, and Modal caps that at 64 KiB + # (ARG_MAX). pallets_jinja/1465's combined.patch alone is >100 KB. enc = base64.b64encode(content.encode()).decode() - sb.exec("bash", "-c", f"echo '{enc}' | base64 -d > {path}") + sb.exec("bash", "-c", f": > {path}") + for i in range(0, len(enc), 32_000): + sb.exec("bash", "-c", f"echo '{enc[i : i + 32_000]}' | base64 -d >> {path}") -def check_feature(repo: str, task: str, feature: str, local_runner: bool = False) -> dict: +def check_feature(repo: str, task: str, feature: str, local_runner: bool = False, backend: str = "modal") -> dict: fd = DATASET / repo / task / feature tests, gold = fd / "tests.patch", fd / "feature.patch" out = {"task": f"{repo}/{task}", "feature": feature} if not (tests.is_file() and gold.is_file()): return {**out, "verdict": "SKIP", "note": "missing tests.patch or feature.patch"} - image = _amd64_ref(get_image_name(repo, int(task.replace("task", "")))) + image = get_image_name(repo, int(task.replace("task", ""))) + if backend == "modal": + image = _amd64_ref(image) # local docker picks the host arch itself started = time.time() - sb = get_backend("modal").create_sandbox(image, timeout=3600) + sb = get_backend(backend).create_sandbox(image, timeout=3600) try: sb.exec("bash", "-c", "mkdir -p /patches") if local_runner: @@ -96,11 +114,14 @@ def check_feature(repo: str, task: str, feature: str, local_runner: bool = False def run(args: str): # Grep the signal out rather than tail blindly: these runners print a long # `git clean` inventory on exit, which pushes the actual error off the end. - r = sb.exec("bash", "-c", - f"bash /usr/local/bin/runner.sh {args} > /tmp/out.log 2>&1; echo RC=$?; " - "grep -iE 'does not apply|failed to apply|error:|FAILED|assert|" - "[0-9]+ (passed|failed)|no tests|collected|panic|cannot' /tmp/out.log " - "| grep -viE 'Removing |Repository (cleaned|restored)' | tail -25") + r = sb.exec( + "bash", + "-c", + f"bash /usr/local/bin/runner.sh {args} > /tmp/out.log 2>&1; echo RC=$?; " + "grep -iE 'does not apply|failed to apply|error:|FAILED|assert|" + "[0-9]+ (passed|failed)|no tests|collected|panic|cannot' /tmp/out.log " + "| grep -viE 'Removing |Repository (cleaned|restored)' | tail -25", + ) body = r.stdout_read() + r.stderr_read() rc = next((int(ln[3:]) for ln in body.splitlines() if ln.startswith("RC=")), -1) return rc, body @@ -109,9 +130,9 @@ def run(args: str): gold_rc, gold_body = run("tests.patch feature.patch") if gold_rc != 0: - verdict = "GOLD_FAILS" # reference contradicts its own tests + verdict = "GOLD_FAILS" # reference contradicts its own tests elif base_rc == 0: - verdict = "PASSES_ON_BASE" # tests do not measure the feature + verdict = "PASSES_ON_BASE" # tests do not measure the feature else: verdict = "OK" return { @@ -133,9 +154,19 @@ def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("task", nargs="?", help="repo/taskN, default all") ap.add_argument("--workers", type=int, default=12) - ap.add_argument("--local-runner", action="store_true", - help="overwrite the image's baked-in runner.sh with the one in dataset/, so " - "runner fixes are exercised without rebuilding and pushing images") + ap.add_argument( + "--backend", + choices=["modal", "docker"], + default="modal", + help="modal = linux/amd64 sandboxes; docker = the local daemon (arm64 on Apple " + "Silicon), so the two together cover both published architectures", + ) + ap.add_argument( + "--local-runner", + action="store_true", + help="overwrite the image's baked-in runner.sh with the one in dataset/, so " + "runner fixes are exercised without rebuilding and pushing images", + ) args = ap.parse_args() feats = [ @@ -145,27 +176,30 @@ def main() -> None: ] if not feats: raise SystemExit(f"no features matched {args.task!r}") - print(f"checking {len(feats)} features, {args.workers} sandboxes at a time\n", flush=True) + report = report_path(args.backend, args.local_runner) + print(f"checking {len(feats)} features on {args.backend}, {args.workers} sandboxes at a time\n", flush=True) results, done = [], 0 with ThreadPoolExecutor(max_workers=args.workers) as pool: - futures = {pool.submit(check_feature, *f, args.local_runner): f for f in feats} + futures = {pool.submit(check_feature, *f, args.local_runner, args.backend): f for f in feats} for fut in as_completed(futures): r = fut.result() results.append(r) done += 1 if r["verdict"] != "OK": - print(f" [{done}/{len(feats)}] {r['verdict']:15s} {r['task']} {r['feature']}" - f" {r.get('note','')}", flush=True) + print( + f" [{done}/{len(feats)}] {r['verdict']:15s} {r['task']} {r['feature']} {r.get('note', '')}", + flush=True, + ) elif done % 20 == 0: print(f" [{done}/{len(feats)}] ...", flush=True) - (REPORT_LOCAL if args.local_runner else REPORT).write_text(json.dumps(sorted(results, key=lambda r: (r["task"], r["feature"])), indent=1)) + report.write_text(json.dumps(sorted(results, key=lambda r: (r["task"], r["feature"])), indent=1)) tally: dict[str, int] = {} for r in results: tally[r["verdict"]] = tally.get(r["verdict"], 0) + 1 print("\n" + " · ".join(f"{k} {v}" for k, v in sorted(tally.items()))) - print(f"report -> {REPORT_LOCAL if args.local_runner else REPORT}") + print(f"report -> {report}") if __name__ == "__main__":