From 75d097ec1f749dafa3ab37f9d861fc9ed04f7296 Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 17:53:40 +0200 Subject: [PATCH 1/8] Support config-specific task questions --- src/tasksource/access.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tasksource/access.py b/src/tasksource/access.py index 8241672..4ffa22f 100644 --- a/src/tasksource/access.py +++ b/src/tasksource/access.py @@ -204,6 +204,8 @@ def load_preprocessing(tasks=tasks, **kwargs): for c in 'dataset_name','config_name': if not isinstance(getattr(preprocessing,c), str): setattr(preprocessing,c,getattr(y,c)) + if isinstance(preprocessing.question, dict): + preprocessing.question = preprocessing.question.get(preprocessing.config_name) preprocessing.dataset_name = CANONICAL.get(preprocessing.dataset_name, preprocessing.dataset_name) return preprocessing From 86777cf57fef7390edbe5529910900e3bf6cd07e Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 17:54:19 +0200 Subject: [PATCH 2/8] Make ambiguous Jev task predicates explicit --- src/tasksource/tasks.py | 52 +++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/tasksource/tasks.py b/src/tasksource/tasks.py index 6bf03ad..c0a75aa 100755 --- a/src/tasksource/tasks.py +++ b/src/tasksource/tasks.py @@ -427,7 +427,8 @@ def orient(row, index): "text", labels="label_text", dataset_name="SetFit/toxic_conversations") -turingbench = Classification("Generation",labels="label", +turingbench = Classification("Generation", labels="label", + question="Who or what generated this text?", dataset_name="csv", task_id="TuringBench", load_dataset_kwargs={"data_files": { "train": "hf://datasets/jana4/turingbench-humanized/TuringBench/AA/train.csv", @@ -735,7 +736,18 @@ def pre_process(dataset): 'political-media-audience', 'political-media-bias', 'political-media-message', - 'text_emotion'] + 'text_emotion'], + question={ + "sentiment_nuclear_power": "What sentiment or stance does the tweet express about nuclear energy?", + "tweet_global_warming": "Does the tweet indicate that the author believes global warming is occurring?", + "airline-sentiment": "What sentiment does the tweet express about the airline?", + "corporate-messaging": "What type of corporate social-media message is this?", + "economic-news": "Is this article relevant to the U.S. economy?", + "political-media-audience": "Is this political message aimed at a constituency or a national audience?", + "political-media-bias": "Is this political message partisan or neutral?", + "political-media-message": "What type of political message is this?", + "text_emotion": "What emotion does the text express?", + }, ) def _ethics_binary_label(x): @@ -1260,20 +1272,29 @@ def _civil(attribute, negative, positive, flag): cloth = MultipleChoice("sentence", choices_list=lambda x:[x["answer"]]+x["distractors"],labels=constant(0), dataset_name="AndyChiang/cloth") dgen = MultipleChoice("sentence", choices_list=lambda x:[x["answer"]]+x["distractors"],labels=constant(0), dataset_name="AndyChiang/dgen") -i2d2 = Classification("sentence1",labels=name('label',['False','True']), dataset_name="tasksource/I2D2") +i2d2 = Classification( + "sentence1", labels=name("label", ["False", "True"]), + question="Is this a plausible commonsense statement?", + dataset_name="tasksource/I2D2") arg_me = Classification( - 'argument', 'conclusion', 'stance', dataset_name="webis/args_me", task_id="args_me", + "argument", "conclusion", "stance", + question="What stance does the argument take toward the conclusion?", + dataset_name="webis/args_me", task_id="args_me", load_dataset_kwargs=dict(revision=PARQUET, data_dir="corpus")) # one argument per row valueeval_stance = Classification( - "Premise", "Conclusion", "Stance", dataset_name="csv", - task_id="Touche23-ValueEval", + "Premise", "Conclusion", "Stance", + question="Does the premise argue in favor of or against the conclusion?", + dataset_name="csv", task_id="Touche23-ValueEval", load_dataset_kwargs={"data_files": { "train": "https://zenodo.org/records/7879430/files/arguments-training.tsv", "validation": "https://zenodo.org/records/7879430/files/arguments-validation.tsv", "test": "https://zenodo.org/records/7879430/files/arguments-test.tsv", }, "delimiter": "\t"}) -starcon = Classification('argument','topic','label',dataset_name="tasksource/starcon") +starcon = Classification( + "argument", "topic", "label", + question="What stance does the argument take toward the topic?", + dataset_name="tasksource/starcon") banking77 = Classification("text",labels="label",dataset_name="legacy-datasets/banking77") @@ -1306,12 +1327,17 @@ def _civil(attribute, negative, positive, flag): cladder = Classification("given_info", "question", "answer",dataset_name="tasksource/cladder") -subjectivity = Classification("Sentence",labels=lambda x: {"OBJ": "objective", "SUBJ": "subjective"}[x["Label"]],dataset_name="tasksource/subjectivity") +subjectivity = Classification( + "Sentence", labels=lambda x: {"OBJ": "objective", "SUBJ": "subjective"}[x["Label"]], + question="Is the sentence objective or subjective?", + dataset_name="tasksource/subjectivity") -moh = Classification("context","expression","label", dataset_name="tasksource/MOH") -vuac = Classification("context","expression","label", dataset_name="tasksource/VUAC") +_metaphor_question = "Is the target expression used literally or metaphorically?" +moh = Classification("context", "expression", "label", question=_metaphor_question, dataset_name="tasksource/MOH") +vuac = Classification("context", "expression", "label", question=_metaphor_question, dataset_name="tasksource/VUAC") trofi = Classification( - "context", "expression", "label", dataset_name="parquet", task_id="TroFi", + "context", "expression", "label", question=_metaphor_question, + dataset_name="parquet", task_id="TroFi", load_dataset_kwargs={"data_files": { "train": "hf://datasets/tasksource/TroFi/data/train-00000-of-00001-67b67b8474db644d.parquet", "test": "hf://datasets/tasksource/TroFi/data/test-00000-of-00001-a467035ce73d87fe.parquet", @@ -1546,7 +1572,9 @@ def _support_shift_name(shift): label_values={shift: _support_shift_name(shift) for shift in range(-2, 6)}) -ambigNQ = Classification("question",labels=lambda x:{True:"ambiguous", False:"not ambiguous"}.get(x["ambig"]), +ambigNQ = Classification( + "question", labels=lambda x: {True: "ambiguous", False: "not ambiguous"}.get(x["ambig"]), + question="Is the question ambiguous?", dataset_name="erbacher/AmbigNQ-clarifying-question") siga_nli = Classification("premise","statement","label",dataset_name="tasksource/SIGA-nli") From efd45d02257da24e78aee3e55600145740e3d964 Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 17:54:26 +0200 Subject: [PATCH 3/8] Mark XGLUE WPR as ordinal --- src/tasksource/multilingual_tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tasksource/multilingual_tasks.py b/src/tasksource/multilingual_tasks.py index 06825b5..a5c964d 100644 --- a/src/tasksource/multilingual_tasks.py +++ b/src/tasksource/multilingual_tasks.py @@ -85,7 +85,9 @@ def _x_fact_labels(dataset): dataset_name="tasksource/xglue", config_name="qadsm") xglue___qam = Classification('question','answer','label', question="Does the passage answer the query?", dataset_name="tasksource/xglue", config_name="qam") -xglue___wpr = Classification('query','web_page_snippet','relavance_label', question="How relevant is the web page to the query?", +xglue___wpr = Classification( + "query", "web_page_snippet", "relavance_label", + question="How relevant is the web page to the query?", ordinal=True, dataset_name="tasksource/xglue", config_name="wpr") # relavance_label : sic xlwic = Classification( From 4e2fdd8cdfcfad6163119c4f8ed3624cbdef5d6d Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 17:54:59 +0200 Subject: [PATCH 4/8] Bound complete Jev request lengths --- scripts/build_jev_dataset.py | 78 +++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/scripts/build_jev_dataset.py b/scripts/build_jev_dataset.py index ec9d78d..8a8f496 100644 --- a/scripts/build_jev_dataset.py +++ b/scripts/build_jev_dataset.py @@ -88,6 +88,25 @@ def normalized_split(split): return "dev" if split == "validation" else split +def filter_request_lengths(rows, budget): + """Drop rows whose complete rendered request exceeds the training context budget.""" + if not budget.max_tokens: + return rows, 0 + + def fits(row): + question = { + "question_id": row.get("question_id", "decision"), + "kind": row["kind"], + "question": row["question"], + "options": row["options"], + } + return budget.fits(row["state"], [question]) + + before = len(rows) + rows = rows.filter(fits) + return rows, before - len(rows) + + def slug(task_id): digest = hashlib.sha1(task_id.encode("utf-8")).hexdigest()[:10] readable = "".join(c if c.isalnum() else "-" for c in task_id).strip("-")[:70] @@ -98,7 +117,8 @@ def slug(task_id): # never mixes shards built with different settings. The code state is recorded per shard # (report "code") but not enforced: any commit would otherwise invalidate every shard. SHARD_PARAMETERS = ("max_rows", "max_rows_eval", "noul_rate", "score_rate", "permutation_rate", "prompt_rate", - "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items") + "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items", + "max_request_tokens", "request_tokenizer") def _git(*arguments): @@ -928,6 +948,7 @@ def build(args): if args.reuse_incompatible_shards: completed |= stale packing_budget = LengthBudget(args.pack_max_tokens, args.pack_tokenizer) + request_budget = LengthBudget(args.max_request_tokens, args.request_tokenizer) print(f"Selected {len(tasks)} tasks; {len(completed)} already complete", flush=True) manifest_path = output / "build-manifest.json" manifest = build_manifest(args, tasks) @@ -963,31 +984,31 @@ def build(args): else: dataset = load_jev_task(row, max_rows, max_rows_eval, loaded, file_pins) split_rows = {} + request_length_dropped = {} for split, split_dataset in dataset.items(): - if row.task_type == "NativeJev": - partial[split] = data_dir / f".{split}-{slug(task_id)}.parquet.partial" - split_dataset.to_parquet(partial[split]) - split_rows[split] = len(split_dataset) - continue - split_dataset = split_dataset.map( - to_training_row, - with_indices=True, - fn_kwargs={"task_id": task_id, "split": split}, - remove_columns=split_dataset.column_names, - ) - pack_audit = [] - split_dataset = add_packed_classification( - split_dataset, row.task_type, rate=args.pack_rate, - budget=packing_budget, max_items=args.pack_max_items, - audit=pack_audit, - ) - write_pack_audit(output, split, task_id, pack_audit) - if row.task_type != "SoftLabeling": # a distribution's questions are asked as authored - split_dataset = augment_jev_internal( - split_dataset, args.noul_rate, args.score_rate, - args.permutation_rate, args.prompt_rate, - args.paired_format_rate, + if row.task_type != "NativeJev": + split_dataset = split_dataset.map( + to_training_row, + with_indices=True, + fn_kwargs={"task_id": task_id, "split": split}, + remove_columns=split_dataset.column_names, ) + pack_audit = [] + split_dataset = add_packed_classification( + split_dataset, row.task_type, rate=args.pack_rate, + budget=packing_budget, max_items=args.pack_max_items, + audit=pack_audit, + ) + write_pack_audit(output, split, task_id, pack_audit) + if row.task_type != "SoftLabeling": # a distribution's questions are asked as authored + split_dataset = augment_jev_internal( + split_dataset, args.noul_rate, args.score_rate, + args.permutation_rate, args.prompt_rate, + args.paired_format_rate, + ) + split_dataset, dropped = filter_request_lengths(split_dataset, request_budget) + if dropped: + request_length_dropped[split] = dropped partial[split] = data_dir / f".{split}-{slug(task_id)}.parquet.partial" split_dataset.to_parquet(partial[split]) split_rows[split] = len(split_dataset) @@ -1002,6 +1023,7 @@ def build(args): "task_type": row.task_type, "status": "ok", "rows": split_rows, + "request_length_dropped": request_length_dropped, "fingerprint": fingerprint, "code": code_state(), "revisions": pins, @@ -1128,6 +1150,14 @@ def parse_args(): help="Hugging Face tokenizer for exact budgets; default is a conservative UTF-8 byte bound.", ) parser.add_argument("--pack-max-items", type=int, default=4) + parser.add_argument( + "--max-request-tokens", type=int, default=32_768, + help="Maximum complete rendered request length; without --request-tokenizer, UTF-8 bytes are a conservative token upper bound. Set 0 to disable.", + ) + parser.add_argument( + "--request-tokenizer", + help="Hugging Face tokenizer for exact complete-request length filtering.", + ) parser.add_argument("--finalize", action="store_true") parser.add_argument( "--finalize-only", action="store_true", From b039c6f801bcd191162ef2cd657b978d9c71aa2d Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 17:55:32 +0200 Subject: [PATCH 5/8] Test Jev audit fixes --- tests/test_build_provenance.py | 46 ++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py index ec7e7eb..f5a738c 100644 --- a/tests/test_build_provenance.py +++ b/tests/test_build_provenance.py @@ -8,14 +8,19 @@ from tasksource import list_tasks, task_provenance from tasksource.access import load_preprocessing +from tasksource.jev.length import LengthBudget from tasksource.jev.recast import render_typed_decision, render_typed_decision_group +from tasksource.multilingual_tasks import xglue___wpr from tasksource.tasks import _intent_grasp_keep -from scripts.build_jev_dataset import build_fingerprint, read_completed, slug, source_provenance +from scripts.build_jev_dataset import ( + build_fingerprint, filter_request_lengths, read_completed, slug, source_provenance, +) def _args(**overrides): values = dict(max_rows=1000, max_rows_eval=100, noul_rate=0.05, score_rate=0.0, permutation_rate=0.05, prompt_rate=0.05, paired_format_rate=0.05, pack_rate=0.1, pack_max_tokens=4096, - pack_tokenizer=None, pack_max_items=4, output=Path("a"), upload=False) + pack_tokenizer=None, pack_max_items=4, max_request_tokens=32768, + request_tokenizer=None, output=Path("a"), upload=False) return argparse.Namespace(**{**values, **overrides}) @@ -23,6 +28,7 @@ class ResumeTest(unittest.TestCase): def test_fingerprint_tracks_shard_settings_only(self): base = build_fingerprint(_args()) self.assertNotEqual(base, build_fingerprint(_args(max_rows=30000))) + self.assertNotEqual(base, build_fingerprint(_args(max_request_tokens=16384))) # where the output goes or whether it uploads does not change shard contents self.assertEqual(base, build_fingerprint(_args(output=Path("b"), upload=True))) @@ -68,6 +74,31 @@ def test_group_keeps_each_kind(self): self.assertEqual({qid: q["type"] for qid, q in request["questions"].items()}, {"a": "score", "b": "choice"}) +class RequestLengthBudgetTest(unittest.TestCase): + def test_filters_complete_rendered_requests(self): + from datasets import Dataset + + row = { + "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0], + "state": "short", "question": "Is this valid?", "source": "source", + "variant": "direct", "split": "train", + } + rows = Dataset.from_list([row, {**row, "id": "long", "state": "x" * 2000}]) + filtered, dropped = filter_request_lengths(rows, LengthBudget(512, overhead=0)) + self.assertEqual((len(filtered), dropped, filtered[0]["id"]), (1, 1, "x")) + + def test_zero_budget_disables_filter(self): + from datasets import Dataset + + rows = Dataset.from_list([{ + "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0], + "state": "x" * 2000, "question": "Q?", "source": "source", + "variant": "direct", "split": "train", + }]) + filtered, dropped = filter_request_lengths(rows, LengthBudget(0)) + self.assertEqual((len(filtered), dropped), (1, 0)) + + class CatalogApiTest(unittest.TestCase): def test_intent_grasp_answer_bounds(self): row = lambda index: {"answer_index": [index], "options": ["a", "b", "c"], @@ -83,6 +114,17 @@ def test_list_tasks_accepts_lists_and_returns_copies(self): self.assertEqual(len(list_tasks()), full) self.assertNotIn("extra", list_tasks().columns) + def test_config_specific_questions_and_ordinal_metadata(self): + self.assertEqual( + load_preprocessing(id="crowdflower/economic-news").question, + "Is this article relevant to the U.S. economy?", + ) + self.assertEqual( + load_preprocessing(id="crowdflower/tweet_global_warming").question, + "Does the tweet indicate that the author believes global warming is occurring?", + ) + self.assertTrue(xglue___wpr.ordinal) + def test_lookup_errors(self): with self.assertRaises(KeyError): load_preprocessing(id="no-such-task") From 8cb5d13d861d37a2532b4e3778300c88f9b5c807 Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 18:00:03 +0200 Subject: [PATCH 6/8] Use a non-destructive default request size cap --- scripts/build_jev_dataset.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/scripts/build_jev_dataset.py b/scripts/build_jev_dataset.py index 8a8f496..e45b26c 100644 --- a/scripts/build_jev_dataset.py +++ b/scripts/build_jev_dataset.py @@ -32,7 +32,7 @@ ) from tasksource.jev import procedural from tasksource.jev.derived import VARIANT as PACKED_VARIANT, add_packed_classification, packed_items -from tasksource.jev.length import LengthBudget +from tasksource.jev.length import LengthBudget, render_request from tasksource.jev.options import gold_position_violations @@ -88,9 +88,9 @@ def normalized_split(split): return "dev" if split == "validation" else split -def filter_request_lengths(rows, budget): - """Drop rows whose complete rendered request exceeds the training context budget.""" - if not budget.max_tokens: +def filter_request_lengths(rows, max_bytes=131_072, budget=None): + """Drop rows whose complete rendered request exceeds byte or exact-token budgets.""" + if not max_bytes and budget is None: return rows, 0 def fits(row): @@ -100,7 +100,10 @@ def fits(row): "question": row["question"], "options": row["options"], } - return budget.fits(row["state"], [question]) + questions = [question] + if max_bytes and len(render_request(row["state"], questions).encode("utf-8")) > max_bytes: + return False + return budget is None or budget.fits(row["state"], questions) before = len(rows) rows = rows.filter(fits) @@ -118,7 +121,7 @@ def slug(task_id): # (report "code") but not enforced: any commit would otherwise invalidate every shard. SHARD_PARAMETERS = ("max_rows", "max_rows_eval", "noul_rate", "score_rate", "permutation_rate", "prompt_rate", "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items", - "max_request_tokens", "request_tokenizer") + "max_request_bytes", "max_request_tokens", "request_tokenizer") def _git(*arguments): @@ -948,7 +951,10 @@ def build(args): if args.reuse_incompatible_shards: completed |= stale packing_budget = LengthBudget(args.pack_max_tokens, args.pack_tokenizer) - request_budget = LengthBudget(args.max_request_tokens, args.request_tokenizer) + request_budget = ( + LengthBudget(args.max_request_tokens, args.request_tokenizer) + if args.request_tokenizer and args.max_request_tokens else None + ) print(f"Selected {len(tasks)} tasks; {len(completed)} already complete", flush=True) manifest_path = output / "build-manifest.json" manifest = build_manifest(args, tasks) @@ -1006,7 +1012,8 @@ def build(args): args.permutation_rate, args.prompt_rate, args.paired_format_rate, ) - split_dataset, dropped = filter_request_lengths(split_dataset, request_budget) + split_dataset, dropped = filter_request_lengths( + split_dataset, args.max_request_bytes, request_budget) if dropped: request_length_dropped[split] = dropped partial[split] = data_dir / f".{split}-{slug(task_id)}.parquet.partial" @@ -1150,13 +1157,17 @@ def parse_args(): help="Hugging Face tokenizer for exact budgets; default is a conservative UTF-8 byte bound.", ) parser.add_argument("--pack-max-items", type=int, default=4) + parser.add_argument( + "--max-request-bytes", type=int, default=131_072, + help="Maximum UTF-8 size of a complete rendered request. Set 0 to disable.", + ) parser.add_argument( "--max-request-tokens", type=int, default=32_768, - help="Maximum complete rendered request length; without --request-tokenizer, UTF-8 bytes are a conservative token upper bound. Set 0 to disable.", + help="Exact token cap when --request-tokenizer is supplied. Set 0 to disable.", ) parser.add_argument( "--request-tokenizer", - help="Hugging Face tokenizer for exact complete-request length filtering.", + help="Hugging Face tokenizer used for the optional exact token cap.", ) parser.add_argument("--finalize", action="store_true") parser.add_argument( From cce10c6f31e89203dfb3b41802d9289088988cf4 Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 18:00:22 +0200 Subject: [PATCH 7/8] Test byte and exact-token request limits --- tests/test_build_provenance.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py index f5a738c..d2cd5bc 100644 --- a/tests/test_build_provenance.py +++ b/tests/test_build_provenance.py @@ -19,8 +19,8 @@ def _args(**overrides): values = dict(max_rows=1000, max_rows_eval=100, noul_rate=0.05, score_rate=0.0, permutation_rate=0.05, prompt_rate=0.05, paired_format_rate=0.05, pack_rate=0.1, pack_max_tokens=4096, - pack_tokenizer=None, pack_max_items=4, max_request_tokens=32768, - request_tokenizer=None, output=Path("a"), upload=False) + pack_tokenizer=None, pack_max_items=4, max_request_bytes=131072, + max_request_tokens=32768, request_tokenizer=None, output=Path("a"), upload=False) return argparse.Namespace(**{**values, **overrides}) @@ -28,6 +28,7 @@ class ResumeTest(unittest.TestCase): def test_fingerprint_tracks_shard_settings_only(self): base = build_fingerprint(_args()) self.assertNotEqual(base, build_fingerprint(_args(max_rows=30000))) + self.assertNotEqual(base, build_fingerprint(_args(max_request_bytes=65536))) self.assertNotEqual(base, build_fingerprint(_args(max_request_tokens=16384))) # where the output goes or whether it uploads does not change shard contents self.assertEqual(base, build_fingerprint(_args(output=Path("b"), upload=True))) @@ -84,7 +85,7 @@ def test_filters_complete_rendered_requests(self): "variant": "direct", "split": "train", } rows = Dataset.from_list([row, {**row, "id": "long", "state": "x" * 2000}]) - filtered, dropped = filter_request_lengths(rows, LengthBudget(512, overhead=0)) + filtered, dropped = filter_request_lengths(rows, max_bytes=512) self.assertEqual((len(filtered), dropped, filtered[0]["id"]), (1, 1, "x")) def test_zero_budget_disables_filter(self): @@ -95,9 +96,26 @@ def test_zero_budget_disables_filter(self): "state": "x" * 2000, "question": "Q?", "source": "source", "variant": "direct", "split": "train", }]) - filtered, dropped = filter_request_lengths(rows, LengthBudget(0)) + filtered, dropped = filter_request_lengths(rows, max_bytes=0) self.assertEqual((len(filtered), dropped), (1, 0)) + def test_optional_exact_token_budget(self): + from datasets import Dataset + + class Tokenizer: + def encode(self, text, add_special_tokens=False): + return list(text) + + row = { + "id": "x", "kind": "choice", "options": ["yes", "no"], "target": [1.0, 0.0], + "state": "x" * 300, "question": "Q?", "source": "source", + "variant": "direct", "split": "train", + } + rows = Dataset.from_list([row]) + filtered, dropped = filter_request_lengths( + rows, max_bytes=10_000, budget=LengthBudget(128, Tokenizer(), overhead=0)) + self.assertEqual((len(filtered), dropped), (0, 1)) + class CatalogApiTest(unittest.TestCase): def test_intent_grasp_answer_bounds(self): From 86ecf997d8aed24bd7339765930437efef491e34 Mon Sep 17 00:00:00 2001 From: sileod Date: Fri, 25 Sep 2026 18:00:50 +0200 Subject: [PATCH 8/8] Clarify nuclear sentiment question --- src/tasksource/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasksource/tasks.py b/src/tasksource/tasks.py index c0a75aa..c848a61 100755 --- a/src/tasksource/tasks.py +++ b/src/tasksource/tasks.py @@ -738,7 +738,7 @@ def pre_process(dataset): 'political-media-message', 'text_emotion'], question={ - "sentiment_nuclear_power": "What sentiment or stance does the tweet express about nuclear energy?", + "sentiment_nuclear_power": "What is the tweet's sentiment toward nuclear energy, or is it unrelated?", "tweet_global_warming": "Does the tweet indicate that the author believes global warming is occurring?", "airline-sentiment": "What sentiment does the tweet express about the airline?", "corporate-messaging": "What type of corporate social-media message is this?",