Fix Jev task questions, ordinal typing, and extreme request lengths

scripts/build_jev_dataset.py

@@ -32,7 +32,7 @@
 )
 from tasksource.jev import procedural
 from tasksource.jev.derived import VARIANT as PACKED_VARIANT, add_packed_classification, packed_items
-from tasksource.jev.length import LengthBudget
+from tasksource.jev.length import LengthBudget, render_request
 from tasksource.jev.options import gold_position_violations
 
 
@@ -88,6 +88,28 @@ def normalized_split(split):
     return "dev" if split == "validation" else split
 
 
+def filter_request_lengths(rows, max_bytes=131_072, budget=None):
+    """Drop rows whose complete rendered request exceeds byte or exact-token budgets."""
+    if not max_bytes and budget is None:
+        return rows, 0
+
+    def fits(row):
+        question = {
+            "question_id": row.get("question_id", "decision"),
+            "kind": row["kind"],
+            "question": row["question"],
+            "options": row["options"],
+        }
+        questions = [question]
+        if max_bytes and len(render_request(row["state"], questions).encode("utf-8")) > max_bytes:
+            return False
+        return budget is None or budget.fits(row["state"], questions)
+
+    before = len(rows)
+    rows = rows.filter(fits)
+    return rows, before - len(rows)
+
+
 def slug(task_id):
     digest = hashlib.sha1(task_id.encode("utf-8")).hexdigest()[:10]
     readable = "".join(c if c.isalnum() else "-" for c in task_id).strip("-")[:70]
@@ -98,7 +120,8 @@ def slug(task_id):
 # never mixes shards built with different settings. The code state is recorded per shard
 # (report "code") but not enforced: any commit would otherwise invalidate every shard.
 SHARD_PARAMETERS = ("max_rows", "max_rows_eval", "noul_rate", "score_rate", "permutation_rate", "prompt_rate",
-                    "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items")
+                    "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items",
+                    "max_request_bytes", "max_request_tokens", "request_tokenizer")
 
 
 def _git(*arguments):
@@ -928,6 +951,10 @@ def build(args):
     if args.reuse_incompatible_shards:
         completed |= stale
     packing_budget = LengthBudget(args.pack_max_tokens, args.pack_tokenizer)
+    request_budget = (
+        LengthBudget(args.max_request_tokens, args.request_tokenizer)
+        if args.request_tokenizer and args.max_request_tokens else None
+    )
     print(f"Selected {len(tasks)} tasks; {len(completed)} already complete", flush=True)
     manifest_path = output / "build-manifest.json"
     manifest = build_manifest(args, tasks)
@@ -963,31 +990,32 @@ def build(args):
             else:
                 dataset = load_jev_task(row, max_rows, max_rows_eval, loaded, file_pins)
             split_rows = {}
+            request_length_dropped = {}
             for split, split_dataset in dataset.items():
-                if row.task_type == "NativeJev":
-                    partial[split] = data_dir / f".{split}-{slug(task_id)}.parquet.partial"
-                    split_dataset.to_parquet(partial[split])
-                    split_rows[split] = len(split_dataset)
-                    continue
-                split_dataset = split_dataset.map(
-                    to_training_row,
-                    with_indices=True,
-                    fn_kwargs={"task_id": task_id, "split": split},
-                    remove_columns=split_dataset.column_names,
-                )
-                pack_audit = []
-                split_dataset = add_packed_classification(
-                    split_dataset, row.task_type, rate=args.pack_rate,
-                    budget=packing_budget, max_items=args.pack_max_items,
-                    audit=pack_audit,
-                )
-                write_pack_audit(output, split, task_id, pack_audit)
-                if row.task_type != "SoftLabeling":  # a distribution's questions are asked as authored
-                    split_dataset = augment_jev_internal(
-                        split_dataset, args.noul_rate, args.score_rate,
-                        args.permutation_rate, args.prompt_rate,
-                        args.paired_format_rate,
+                if row.task_type != "NativeJev":
+                    split_dataset = split_dataset.map(
+                        to_training_row,
+                        with_indices=True,
+                        fn_kwargs={"task_id": task_id, "split": split},
+                        remove_columns=split_dataset.column_names,
                     )
+                    pack_audit = []
+                    split_dataset = add_packed_classification(
+                        split_dataset, row.task_type, rate=args.pack_rate,
+                        budget=packing_budget, max_items=args.pack_max_items,
+                        audit=pack_audit,
+                    )
+                    write_pack_audit(output, split, task_id, pack_audit)
+                    if row.task_type != "SoftLabeling":  # a distribution's questions are asked as authored
+                        split_dataset = augment_jev_internal(
+                            split_dataset, args.noul_rate, args.score_rate,
+                            args.permutation_rate, args.prompt_rate,
+                            args.paired_format_rate,
+                        )
+                split_dataset, dropped = filter_request_lengths(
+                    split_dataset, args.max_request_bytes, request_budget)
+                if dropped:
+                    request_length_dropped[split] = dropped
                 partial[split] = data_dir / f".{split}-{slug(task_id)}.parquet.partial"
                 split_dataset.to_parquet(partial[split])
                 split_rows[split] = len(split_dataset)
@@ -1002,6 +1030,7 @@ def build(args):
                 "task_type": row.task_type,
                 "status": "ok",
                 "rows": split_rows,
+                "request_length_dropped": request_length_dropped,
                 "fingerprint": fingerprint,
                 "code": code_state(),
                 "revisions": pins,
@@ -1128,6 +1157,18 @@ def parse_args():
         help="Hugging Face tokenizer for exact budgets; default is a conservative UTF-8 byte bound.",
     )
     parser.add_argument("--pack-max-items", type=int, default=4)
+    parser.add_argument(
+        "--max-request-bytes", type=int, default=131_072,
+        help="Maximum UTF-8 size of a complete rendered request. Set 0 to disable.",
+    )
+    parser.add_argument(
+        "--max-request-tokens", type=int, default=32_768,
+        help="Exact token cap when --request-tokenizer is supplied. Set 0 to disable.",
+    )
+    parser.add_argument(
+        "--request-tokenizer",
+        help="Hugging Face tokenizer used for the optional exact token cap.",
+    )
     parser.add_argument("--finalize", action="store_true")
     parser.add_argument(
         "--finalize-only", action="store_true",

src/tasksource/access.py

@@ -204,6 +204,8 @@ def load_preprocessing(tasks=tasks, **kwargs):
     for c in 'dataset_name','config_name':
         if not isinstance(getattr(preprocessing,c), str):
              setattr(preprocessing,c,getattr(y,c))
+    if isinstance(preprocessing.question, dict):
+        preprocessing.question = preprocessing.question.get(preprocessing.config_name)
     preprocessing.dataset_name = CANONICAL.get(preprocessing.dataset_name, preprocessing.dataset_name)
     return preprocessing
 

src/tasksource/multilingual_tasks.py

@@ -85,7 +85,9 @@ def _x_fact_labels(dataset):
     dataset_name="tasksource/xglue", config_name="qadsm")
 xglue___qam = Classification('question','answer','label', question="Does the passage answer the query?",
     dataset_name="tasksource/xglue", config_name="qam")
-xglue___wpr = Classification('query','web_page_snippet','relavance_label', question="How relevant is the web page to the query?",
+xglue___wpr = Classification(
+    "query", "web_page_snippet", "relavance_label",
+    question="How relevant is the web page to the query?", ordinal=True,
     dataset_name="tasksource/xglue", config_name="wpr") # relavance_label : sic
 
 xlwic = Classification(

src/tasksource/tasks.py

@@ -427,7 +427,8 @@ def orient(row, index):
     "text", labels="label_text",
     dataset_name="SetFit/toxic_conversations")
 
-turingbench = Classification("Generation",labels="label",
+turingbench = Classification("Generation", labels="label",
+    question="Who or what generated this text?",
     dataset_name="csv", task_id="TuringBench",
     load_dataset_kwargs={"data_files": {
         "train": "hf://datasets/jana4/turingbench-humanized/TuringBench/AA/train.csv",
@@ -735,7 +736,18 @@ def pre_process(dataset):
             'political-media-audience',
             'political-media-bias',
             'political-media-message',
-            'text_emotion']
+            'text_emotion'],
+ question={
+     "sentiment_nuclear_power": "What is the tweet's sentiment toward nuclear energy, or is it unrelated?",
+     "tweet_global_warming": "Does the tweet indicate that the author believes global warming is occurring?",
+     "airline-sentiment": "What sentiment does the tweet express about the airline?",
+     "corporate-messaging": "What type of corporate social-media message is this?",
+     "economic-news": "Is this article relevant to the U.S. economy?",
+     "political-media-audience": "Is this political message aimed at a constituency or a national audience?",
+     "political-media-bias": "Is this political message partisan or neutral?",
+     "political-media-message": "What type of political message is this?",
+     "text_emotion": "What emotion does the text express?",
+ },
 )
 
 def _ethics_binary_label(x):
@@ -1260,20 +1272,29 @@ def _civil(attribute, negative, positive, flag):
 cloth = MultipleChoice("sentence", choices_list=lambda x:[x["answer"]]+x["distractors"],labels=constant(0), dataset_name="AndyChiang/cloth")
 dgen  = MultipleChoice("sentence", choices_list=lambda x:[x["answer"]]+x["distractors"],labels=constant(0), dataset_name="AndyChiang/dgen")
 
-i2d2 = Classification("sentence1",labels=name('label',['False','True']), dataset_name="tasksource/I2D2")
+i2d2 = Classification(
+    "sentence1", labels=name("label", ["False", "True"]),
+    question="Is this a plausible commonsense statement?",
+    dataset_name="tasksource/I2D2")
 
 arg_me = Classification(
-    'argument', 'conclusion', 'stance', dataset_name="webis/args_me", task_id="args_me",
+    "argument", "conclusion", "stance",
+    question="What stance does the argument take toward the conclusion?",
+    dataset_name="webis/args_me", task_id="args_me",
     load_dataset_kwargs=dict(revision=PARQUET, data_dir="corpus"))  # one argument per row
 valueeval_stance = Classification(
-    "Premise", "Conclusion", "Stance", dataset_name="csv",
-    task_id="Touche23-ValueEval",
+    "Premise", "Conclusion", "Stance",
+    question="Does the premise argue in favor of or against the conclusion?",
+    dataset_name="csv", task_id="Touche23-ValueEval",
     load_dataset_kwargs={"data_files": {
         "train": "https://zenodo.org/records/7879430/files/arguments-training.tsv",
         "validation": "https://zenodo.org/records/7879430/files/arguments-validation.tsv",
         "test": "https://zenodo.org/records/7879430/files/arguments-test.tsv",
     }, "delimiter": "\t"})
-starcon = Classification('argument','topic','label',dataset_name="tasksource/starcon")
+starcon = Classification(
+    "argument", "topic", "label",
+    question="What stance does the argument take toward the topic?",
+    dataset_name="tasksource/starcon")
 
 banking77 = Classification("text",labels="label",dataset_name="legacy-datasets/banking77")
 
@@ -1306,12 +1327,17 @@ def _civil(attribute, negative, positive, flag):
 
 cladder = Classification("given_info", "question", "answer",dataset_name="tasksource/cladder")
 
-subjectivity = Classification("Sentence",labels=lambda x: {"OBJ": "objective", "SUBJ": "subjective"}[x["Label"]],dataset_name="tasksource/subjectivity")
+subjectivity = Classification(
+    "Sentence", labels=lambda x: {"OBJ": "objective", "SUBJ": "subjective"}[x["Label"]],
+    question="Is the sentence objective or subjective?",
+    dataset_name="tasksource/subjectivity")
 
-moh   = Classification("context","expression","label", dataset_name="tasksource/MOH")
-vuac  = Classification("context","expression","label", dataset_name="tasksource/VUAC")
+_metaphor_question = "Is the target expression used literally or metaphorically?"
+moh = Classification("context", "expression", "label", question=_metaphor_question, dataset_name="tasksource/MOH")
+vuac = Classification("context", "expression", "label", question=_metaphor_question, dataset_name="tasksource/VUAC")
 trofi = Classification(
-    "context", "expression", "label", dataset_name="parquet", task_id="TroFi",
+    "context", "expression", "label", question=_metaphor_question,
+    dataset_name="parquet", task_id="TroFi",
     load_dataset_kwargs={"data_files": {
         "train": "hf://datasets/tasksource/TroFi/data/train-00000-of-00001-67b67b8474db644d.parquet",
         "test": "hf://datasets/tasksource/TroFi/data/test-00000-of-00001-a467035ce73d87fe.parquet",
@@ -1546,7 +1572,9 @@ def _support_shift_name(shift):
     label_values={shift: _support_shift_name(shift) for shift in range(-2, 6)})
 
 
-ambigNQ = Classification("question",labels=lambda x:{True:"ambiguous", False:"not ambiguous"}.get(x["ambig"]),
+ambigNQ = Classification(
+    "question", labels=lambda x: {True: "ambiguous", False: "not ambiguous"}.get(x["ambig"]),
+    question="Is the question ambiguous?",
     dataset_name="erbacher/AmbigNQ-clarifying-question")
 
 siga_nli = Classification("premise","statement","label",dataset_name="tasksource/SIGA-nli")

tests/test_build_provenance.py

@@ -8,21 +8,28 @@
 
 from tasksource import list_tasks, task_provenance
 from tasksource.access import load_preprocessing
+from tasksource.jev.length import LengthBudget
 from tasksource.jev.recast import render_typed_decision, render_typed_decision_group
+from tasksource.multilingual_tasks import xglue___wpr
 from tasksource.tasks import _intent_grasp_keep
-from scripts.build_jev_dataset import build_fingerprint, read_completed, slug, source_provenance
+from scripts.build_jev_dataset import (
+    build_fingerprint, filter_request_lengths, read_completed, slug, source_provenance,
+)
 
 def _args(**overrides):
     values = dict(max_rows=1000, max_rows_eval=100, noul_rate=0.05, score_rate=0.0, permutation_rate=0.05,
                   prompt_rate=0.05, paired_format_rate=0.05, pack_rate=0.1, pack_max_tokens=4096,
-                  pack_tokenizer=None, pack_max_items=4, output=Path("a"), upload=False)
+                  pack_tokenizer=None, pack_max_items=4, max_request_bytes=131072,
+                  max_request_tokens=32768, request_tokenizer=None, output=Path("a"), upload=False)
     return argparse.Namespace(**{**values, **overrides})
 
 
 class ResumeTest(unittest.TestCase):
     def test_fingerprint_tracks_shard_settings_only(self):
         base = build_fingerprint(_args())
         self.assertNotEqual(base, build_fingerprint(_args(max_rows=30000)))
+        self.assertNotEqual(base, build_fingerprint(_args(max_request_bytes=65536)))
+        self.assertNotEqual(base, build_fingerprint(_args(max_request_tokens=16384)))
         # where the output goes or whether it uploads does not change shard contents
         self.assertEqual(base, build_fingerprint(_args(output=Path("b"), upload=True)))
 
@@ -68,6 +75,48 @@ def test_group_keeps_each_kind(self):
         self.assertEqual({qid: q["type"] for qid, q in request["questions"].items()}, {"a": "score", "b": "choice"})
 
 
+class RequestLengthBudgetTest(unittest.TestCase):
+    def test_filters_complete_rendered_requests(self):
+        from datasets import Dataset
+
+        row = {
+            "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0],
+            "state": "short", "question": "Is this valid?", "source": "source",
+            "variant": "direct", "split": "train",
+        }
+        rows = Dataset.from_list([row, {**row, "id": "long", "state": "x" * 2000}])
+        filtered, dropped = filter_request_lengths(rows, max_bytes=512)
+        self.assertEqual((len(filtered), dropped, filtered[0]["id"]), (1, 1, "x"))
+
+    def test_zero_budget_disables_filter(self):
+        from datasets import Dataset
+
+        rows = Dataset.from_list([{
+            "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0],
+            "state": "x" * 2000, "question": "Q?", "source": "source",
+            "variant": "direct", "split": "train",
+        }])
+        filtered, dropped = filter_request_lengths(rows, max_bytes=0)
+        self.assertEqual((len(filtered), dropped), (1, 0))
+
+    def test_optional_exact_token_budget(self):
+        from datasets import Dataset
+
+        class Tokenizer:
+            def encode(self, text, add_special_tokens=False):
+                return list(text)
+
+        row = {
+            "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0],
+            "state": "x" * 300, "question": "Q?", "source": "source",
+            "variant": "direct", "split": "train",
+        }
+        rows = Dataset.from_list([row])
+        filtered, dropped = filter_request_lengths(
+            rows, max_bytes=10_000, budget=LengthBudget(128, Tokenizer(), overhead=0))
+        self.assertEqual((len(filtered), dropped), (0, 1))
+
+
 class CatalogApiTest(unittest.TestCase):
     def test_intent_grasp_answer_bounds(self):
         row = lambda index: {"answer_index": [index], "options": ["a", "b", "c"],
@@ -83,6 +132,17 @@ def test_list_tasks_accepts_lists_and_returns_copies(self):
         self.assertEqual(len(list_tasks()), full)
         self.assertNotIn("extra", list_tasks().columns)
 
+    def test_config_specific_questions_and_ordinal_metadata(self):
+        self.assertEqual(
+            load_preprocessing(id="crowdflower/economic-news").question,
+            "Is this article relevant to the U.S. economy?",
+        )
+        self.assertEqual(
+            load_preprocessing(id="crowdflower/tweet_global_warming").question,
+            "Does the tweet indicate that the author believes global warming is occurring?",
+        )
+        self.assertTrue(xglue___wpr.ordinal)
+
     def test_lookup_errors(self):
         with self.assertRaises(KeyError):
             load_preprocessing(id="no-such-task")