From f8f57402f36c4039bac50c94ef789b2ebdfc2e06 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 25 Sep 2026 08:52:27 +0000
Subject: [PATCH 1/5] Build fingerprints and exact source revisions; typed
 renderer kinds; catalog API fixes

- resume: a build fingerprint (commit, uncommitted src/scripts changes, shard-affecting
  settings) is stored per task; shards from another fingerprint stop the build unless
  --reuse-incompatible-shards; the manifest keeps previous runs and the summary lists the
  shard fingerprints
- revisions: each task's Hub repos are resolved to a commit at the requested revision
  (refs/convert/parquet included) before loading, Hub-repo loads are pinned to it, the
  report records it, and sources.yaml reads it from there instead of today's default branch
- render_typed_decision(_group) render choice, score and noul by kind (score: 2-10 levels)
  instead of always choice
- IntentGrasp drops negative answer indices
- list_tasks accepts a list for excluded and returns a copy of the cached catalog
- task lookup filters directly: KeyError for an unknown task, ValueError when a selector
  matches several

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EVp7VRXPWTGT6HsHHWJghc
---
 scripts/build_jev_dataset.py   | 146 +++++++++++++++++++++++++--------
 src/tasksource/access.py       |  33 ++++++--
 src/tasksource/jev/graded.py   |   9 +-
 src/tasksource/jev/recast.py   |  40 +++++----
 src/tasksource/tasks.py        |  14 ++--
 tests/test_build_provenance.py | 100 ++++++++++++++++++++++
 6 files changed, 271 insertions(+), 71 deletions(-)
 create mode 100644 tests/test_build_provenance.py

diff --git a/scripts/build_jev_dataset.py b/scripts/build_jev_dataset.py
index 702fb73..c70b9bc 100644
--- a/scripts/build_jev_dataset.py
+++ b/scripts/build_jev_dataset.py
@@ -44,10 +44,12 @@
 }
 
 
-def load_jev_task(row, max_rows, max_rows_eval):
+def load_jev_task(row, max_rows, max_rows_eval, revision=None):
+    """``revision`` pins the task's Hub dataset to the commit recorded in the build report."""
     return load_task(
         row.id, recast="jev", multilingual=row.multilingual,
         max_rows=max_rows, max_rows_eval=max_rows_eval,
+        **({"revision": revision} if revision else {}),
     )
 
 
@@ -64,12 +66,12 @@ def load_jev_task(row, max_rows, max_rows_eval):
 )
 
 
-def load_native_task(source_id, max_rows, max_rows_eval):
+def load_native_task(source_id, max_rows, max_rows_eval, revision=None):
     """Sources authored as typed Jev questions, grouped by state (no recast)."""
     if source_id.startswith(graded.SOURCE_PREFIX):
-        rows = graded.load_family(source_id[len(graded.SOURCE_PREFIX):], max_rows, max_rows_eval)
+        rows = graded.load_family(source_id[len(graded.SOURCE_PREFIX):], max_rows, max_rows_eval, revision)
     else:
-        dataset = load_dataset(procedural.REPO_ID, source_id[len(procedural.SOURCE_PREFIX):])
+        dataset = load_dataset(procedural.REPO_ID, source_id[len(procedural.SOURCE_PREFIX):], revision=revision)
         dataset = sample_dataset(dataset, max_rows, max_rows_eval)
         rows = {split: [row for index, example in enumerate(examples)
                         for row in procedural.jev_rows(example, source_id, normalized_split(split), index)]
@@ -90,13 +92,42 @@ def slug(task_id):
     return f"{readable}-{digest}"
 
 
-def read_completed(report_path, data_dir=None):
-    completed = set()
-    if not report_path.exists():
-        return completed
-    for line in report_path.read_text().splitlines():
-        record = json.loads(line)
-        if record["status"] == "ok":
+# Arguments that change what a task's shards contain; with the code state they form the
+# build fingerprint, so a resumed build never mixes shards from different settings or code.
+SHARD_PARAMETERS = ("max_rows", "max_rows_eval", "noul_rate", "score_rate", "permutation_rate", "prompt_rate",
+                    "paired_format_rate", "pack_rate", "pack_max_tokens", "pack_tokenizer", "pack_max_items")
+
+
+def _git(*arguments):
+    root = Path(__file__).resolve().parents[1]
+    return subprocess.run(["git", *arguments], cwd=root, capture_output=True, check=False).stdout
+
+
+def code_state():
+    """Commit plus a hash of uncommitted changes under src/ and scripts/, untracked files included."""
+    root = Path(__file__).resolve().parents[1]
+    digest = hashlib.sha256(_git("diff", "--binary", "HEAD", "--", "src", "scripts"))
+    for name in sorted(_git("ls-files", "--others", "--exclude-standard", "--", "src", "scripts").decode().split()):
+        if not name.endswith((".pyc", ".pyo")):
+            digest.update(name.encode() + b"\0" + (root / name).read_bytes())
+    return {"git_commit": _git("rev-parse", "HEAD").decode().strip(), "uncommitted_sha256": digest.hexdigest()}
+
+
+def build_fingerprint(args, state=None):
+    shard_args = {name: getattr(args, name, None) for name in SHARD_PARAMETERS}
+    payload = json.dumps({**(state or code_state()), **shard_args}, sort_keys=True, default=str)
+    return hashlib.sha256(payload.encode()).hexdigest()[:16]
+
+
+def read_completed(report_path, data_dir=None, fingerprint=None):
+    """Tasks whose latest successful shards exist; with ``fingerprint``, return
+    ``(completed, stale)``: ``stale`` tasks have shards from another fingerprint."""
+    completed, stale = set(), set()
+    if report_path.exists():
+        for line in report_path.read_text().splitlines():
+            record = json.loads(line)
+            if record["status"] != "ok":
+                continue
             task = record["task"]
             splits = record.get("rows", {})
             if data_dir is None or (
@@ -105,8 +136,24 @@ def read_completed(report_path, data_dir=None):
                     for split in splits
                 )
             ):
-                completed.add(task)
-    return completed
+                current = fingerprint is None or record.get("fingerprint") == fingerprint
+                (completed if current else stale).add(task)
+                (stale if current else completed).discard(task)
+    return completed if fingerprint is None else (completed, stale)
+
+
+def resolve_revisions(provenance):
+    """Commit sha of each Hub repo a source loads, at the revision it requests."""
+    api, pins = HfApi(), {}
+    requested = {**({provenance["dataset"]: provenance.get("revision")} if provenance.get("dataset") else {}),
+                 **{repo: provenance.get("data_file_revisions", {}).get(repo)
+                    for repo in provenance.get("data_files_from", [])}}
+    for repo, revision in requested.items():
+        try:
+            pins[repo] = api.dataset_info(repo, revision=revision).sha
+        except Exception:  # gated or offline: loading will tell; the report records no pin
+            pins[repo] = None
+    return pins
 
 
 def build_manifest(args, tasks):
@@ -134,6 +181,7 @@ def git(*arguments):
     return {
         "git_commit": git("rev-parse", "HEAD").decode().strip(),
         "git_diff_sha256": hashlib.sha256(diff).hexdigest(),
+        "fingerprint": build_fingerprint(args),
         "working_tree_changes": status,
         "python": sys.version.split()[0],
         "packages": versions,
@@ -705,7 +753,7 @@ def publish_dataset(
         json.dumps(release_audit, indent=2, ensure_ascii=False) + "\n",
         encoding="utf-8",
     )
-    write_sources_yaml(output / "sources.yaml", release_audit)
+    write_sources_yaml(output / "sources.yaml", release_audit, latest_records(output / "build-report.jsonl"))
     api = HfApi()
     api.upload_file(
         path_or_fileobj=str(output / "README.md"),
@@ -738,10 +786,12 @@ def publish_dataset(
 def source_provenance(source):
     """Hub provenance of one published source (see tasksource.task_provenance)."""
     if source.startswith(procedural.SOURCE_PREFIX):
-        return {"generated": "procedural (tasksource.jev.procedural)"}
+        return {"dataset": procedural.REPO_ID, "config": source[len(procedural.SOURCE_PREFIX):],
+                "generated": "procedural (tasksource.jev.procedural)"}
     if source.startswith(graded.SOURCE_PREFIX):
         family = graded.FAMILIES[source[len(graded.SOURCE_PREFIX):]]
         info = {"dataset": family.dataset, "config": family.config,
+                "revision": (family.load_kwargs or {}).get("revision"),
                 "originals": sorted(set(ORIGINALS.get(family.dataset, [])) - {family.dataset})}
         return {k: v for k, v in info.items() if v}
     if source.startswith("multilingual/"):
@@ -749,17 +799,13 @@ def source_provenance(source):
     return task_provenance(source)
 
 
-def write_sources_yaml(path, release_audit):
-    """Every source in the release with its rows, Hub dataset, revision and originals."""
-    api, revisions = HfApi(), {}
+def write_sources_yaml(path, release_audit, records=None):
+    """Every source in the release with its rows, Hub dataset, revision and originals.
 
-    def revision(repo):
-        if repo not in revisions:
-            try:
-                revisions[repo] = api.dataset_info(repo).sha
-            except Exception:  # gated, renamed or offline: provenance without a pin
-                revisions[repo] = None
-        return revisions[repo]
+    Revisions are the commits each task was loaded from, recorded in the build report
+    (``records``, the latest report record per task); a task built before revisions were
+    recorded is marked ``revisions_unrecorded`` rather than given today's commit."""
+    records = records or {}
 
     splits = release_audit["splits"]
     sources = {}
@@ -769,9 +815,11 @@ def revision(repo):
             info.update(source_provenance(source))
         except KeyError as error:
             info["provenance_error"] = str(error)
-        for repo in [info.get("dataset"), *info.get("data_files_from", [])]:
-            if repo and revision(repo):
-                info.setdefault("revisions", {})[repo] = revision(repo)
+        recorded = (records.get(source) or {}).get("revisions")
+        if recorded:
+            info["revisions"] = {repo: sha for repo, sha in recorded.items() if sha}
+        else:
+            info["revisions_unrecorded"] = True
         sources[source] = info
     try:
         commit = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True,
@@ -877,16 +925,26 @@ def build(args):
             args.prompt_rate, args.paired_format_rate,
         )
     report_path = output / "build-report.jsonl"
-    completed = read_completed(report_path, data_dir)
+    fingerprint = build_fingerprint(args)
+    completed, stale = read_completed(report_path, data_dir, fingerprint)
     tasks = select_tasks(args)
+    stale &= set(tasks.source_id)
+    if stale and not (args.finalize_only or args.reuse_incompatible_shards):
+        raise SystemExit(
+            f"{len(stale)} task shards in {output} come from other code or settings (build fingerprint "
+            f"{fingerprint} differs), e.g. {sorted(stale)[:3]}. Use a fresh --output, or pass "
+            "--reuse-incompatible-shards to keep them (build-manifest.json records their fingerprints).")
+    if args.reuse_incompatible_shards:
+        completed |= stale
     packing_budget = LengthBudget(args.pack_max_tokens, args.pack_tokenizer)
     print(f"Selected {len(tasks)} tasks; {len(completed)} already complete", flush=True)
     manifest_path = output / "build-manifest.json"
-    if not manifest_path.exists():
-        manifest_path.write_text(
-            json.dumps(build_manifest(args, tasks), indent=2, ensure_ascii=False) + "\n",
-            encoding="utf-8",
-        )
+    manifest = build_manifest(args, tasks)
+    if manifest_path.exists():  # keep earlier runs' provenance next to this one's
+        previous = json.loads(manifest_path.read_text(encoding="utf-8"))
+        manifest["previous_runs"] = previous.pop("previous_runs", []) + [
+            {key: previous.get(key) for key in ("git_commit", "git_diff_sha256", "fingerprint", "parameters")}]
+    manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
 
     for position, row in enumerate(
         () if args.finalize_only else tasks.itertuples(index=False), start=1
@@ -903,10 +961,14 @@ def build(args):
             if row.task_type == "TokenClassification":
                 max_rows = max(1, max_rows // 2)
                 max_rows_eval = max(1, max_rows_eval // 2)
+            # pin every Hub repo to the commit its requested revision points to now, and record it
+            provenance = source_provenance(task_id)
+            pins = resolve_revisions(provenance)
+            loaded = pins.get(provenance.get("dataset"))
             if row.task_type == "NativeJev":
-                dataset = load_native_task(task_id, max_rows, max_rows_eval)
+                dataset = load_native_task(task_id, max_rows, max_rows_eval, loaded)
             else:
-                dataset = load_jev_task(row, max_rows, max_rows_eval)
+                dataset = load_jev_task(row, max_rows, max_rows_eval, loaded)
             split_rows = {}
             for split, split_dataset in dataset.items():
                 if row.task_type == "NativeJev":
@@ -940,6 +1002,8 @@ def build(args):
                 "task_type": row.task_type,
                 "status": "ok",
                 "rows": split_rows,
+                "fingerprint": fingerprint,
+                "revisions": pins,
                 "seconds": round(time.time() - started, 3),
             }
             completed.add(task_id)
@@ -981,6 +1045,12 @@ def build(args):
             "outdated_datasets": len(outdated),
             "parquet_files": len(list(data_dir.glob("*.parquet"))),
             "excluded_source_families": list(PUBLISH_EXCLUDED_PREFIXES),
+            # the code/settings fingerprints the selected shards were built with; more than
+            # one means --reuse-incompatible-shards or --finalize-only kept older shards
+            "build_fingerprint": fingerprint,
+            "shard_fingerprints": dict(Counter(
+                latest[task].get("fingerprint") or "unrecorded" for task in sorted(selected & set(latest))
+                if latest[task]["status"] == "ok")),
         }
         (output / "build-summary.json").write_text(
             json.dumps(summary, indent=2) + "\n", encoding="utf-8"
@@ -1016,6 +1086,10 @@ def parse_args():
         "--english-only", action="store_true",
         help="Exclude the multilingual Tasksource catalog.",
     )
+    parser.add_argument(
+        "--reuse-incompatible-shards", action="store_true",
+        help="Resume even when existing shards come from other code or shard settings.",
+    )
     parser.add_argument("--max-rows", type=int, default=30_000)
     parser.add_argument("--max-rows-eval", type=int, default=3_000)
     parser.add_argument(
diff --git a/src/tasksource/access.py b/src/tasksource/access.py
index 353e618..26813f6 100644
--- a/src/tasksource/access.py
+++ b/src/tasksource/access.py
@@ -1,6 +1,8 @@
 from .preprocess import Preprocessing, MultipleChoiceFields, add_question
 from .jev.options import JEV_MAX_MC_OPTIONS
 import re
+from urllib.parse import unquote
+import numpy as np
 import pandas as pd
 from . import tasks, recast as recast_module
 from .metadata import dataset_rank
@@ -44,8 +46,14 @@ def pretty_name(x):
     tn = x.task_name if x.task_name else ""
     return f"{dn}/{cn}/{tn}".replace('//','/').rstrip('/')
 
+def list_tasks(tasks_path=f'{os.path.dirname(__file__)}/tasks.py', multilingual=False, instruct=False, excluded=()):
+    """The task catalog as a DataFrame; ``excluded`` holds substrings of task ids to leave out.
+
+    Each call returns a fresh copy, so callers may edit it without affecting later calls."""
+    return _list_tasks(tasks_path, multilingual, instruct, tuple(excluded)).copy()
+
 @cache
-def list_tasks(tasks_path=f'{os.path.dirname(__file__)}/tasks.py',multilingual=False,instruct=False, excluded=[]):
+def _list_tasks(tasks_path, multilingual, instruct, excluded):
     if multilingual:
         tasks_path=tasks_path.replace('/tasks.py','/multilingual_tasks.py')
     task_order = open(tasks_path).readlines()
@@ -123,16 +131,18 @@ def task_provenance(task_id, multilingual=False):
     if row.empty:
         raise KeyError(f"unknown task: {task_id}")
     row = next(row.itertuples())
-    files = sorted(set(re.findall(r"hf://datasets/([\w.-]+/[\w.-]+)", str(row.mapping.load_dataset_kwargs))))
+    kwargs = row.mapping.load_dataset_kwargs or {}
+    # hf:// data files may pin a ref: hf://datasets/owner/name@refs%2Fconvert%2Fparquet/...
+    refs = {repo: unquote(ref) or None for repo, ref in
+            re.findall(r"hf://datasets/([\w.-]+/[\w.-]+)(?:@([^/\s'\"]+))?", str(kwargs))}
+    files = sorted(refs)
     dataset = None if row.dataset_name in RAW_BUILDERS else row.dataset_name
     originals = sorted({o for key in {dataset, row.id, *files} if key for o in ORIGINALS.get(key, [])} - {dataset})
-    info = {"dataset": dataset, "config": row.config_name or None, "data_files_from": files, "originals": originals}
+    info = {"dataset": dataset, "config": row.config_name or None, "revision": kwargs.get("revision"),
+            "data_files_from": files, "data_file_revisions": {repo: ref for repo, ref in refs.items() if ref},
+            "originals": originals}
     return {k: v for k, v in info.items() if v}
 
-def dict_to_query(d=dict(), **kwargs):
-    d={**d,**kwargs}
-    return '&'.join([f'`{k}`=="{v}"' for k,v in d.items()])
-
 def _format_loader_kwargs(value, **context):
     """Resolve per-config placeholders in generic-builder data_files settings."""
     if isinstance(value, str):
@@ -146,8 +156,13 @@ def _format_loader_kwargs(value, **context):
     return value
 
 def load_preprocessing(tasks=tasks, **kwargs):
-    _tasks_df = list_tasks(multilingual=tasks==lmtasks)
-    y = _tasks_df.copy().query(dict_to_query(**kwargs)).iloc[0]
+    df = list_tasks(multilingual=tasks==lmtasks)
+    matches = df[np.logical_and.reduce([df[k] == v for k, v in kwargs.items()] + [np.ones(len(df), bool)])]
+    if matches.empty:
+        raise KeyError(f"unknown task: {kwargs}")
+    if len(matches) > 1:
+        raise ValueError(f"{kwargs} matches {len(matches)} tasks ({', '.join(matches.id[:5])}...); pass a task id")
+    y = matches.iloc[0]
     preprocessing= copy.copy(getattr(tasks, y.preprocessing_name))
     for c in 'dataset_name','config_name':
         if not isinstance(getattr(preprocessing,c), str):
diff --git a/src/tasksource/jev/graded.py b/src/tasksource/jev/graded.py
index c0d1282..d6234ed 100644
--- a/src/tasksource/jev/graded.py
+++ b/src/tasksource/jev/graded.py
@@ -263,10 +263,13 @@ def jev_rows(example, family, source, split, index):
     return rows
 
 
-def load_family(name, max_rows=None, max_rows_eval=None):
-    """Grouped Jev rows per split, split and sampled like any Tasksource source."""
+def load_family(name, max_rows=None, max_rows_eval=None, revision=None):
+    """Grouped Jev rows per split, split and sampled like any Tasksource source.
+
+    ``revision`` pins the source commit, overriding the family's requested ref."""
     family = FAMILIES[name]
-    dataset = DatasetDict(load_dataset(family.dataset, family.config, **(family.load_kwargs or {})))
+    kwargs = {**(family.load_kwargs or {}), **({"revision": revision} if revision else {})}
+    dataset = DatasetDict(load_dataset(family.dataset, family.config, **kwargs))
     if family.dedupe:
         dataset = DatasetDict({split: rows.select(
             rows.to_pandas().drop_duplicates(family.dedupe).index.tolist()) for split, rows in dataset.items()})
diff --git a/src/tasksource/jev/recast.py b/src/tasksource/jev/recast.py
index 938a095..3c56744 100644
--- a/src/tasksource/jev/recast.py
+++ b/src/tasksource/jev/recast.py
@@ -271,22 +271,34 @@ def convert_batch(batch, indices):
     return converted
 
 
-def render_typed_decision(example, question_id="decision", model=None):
-    """Render one canonical Jev row as a System One choice request."""
+MAX_SCORE_LEVELS = 10
+
+
+def _typed_question(example, instructions):
+    """One System One question from a canonical row; ``kind`` defaults to ``choice``."""
+    kind = example.get("kind") or "choice"
+    if kind == "noul":
+        return {"type": "noul", "instructions": instructions}
     criteria = list(example["criteria"])
     if len(criteria) != len(set(criteria)):
-        raise ValueError("System One choice criterion names must be unique")
+        raise ValueError("System One criterion names must be unique")
+    if kind == "choice":
+        return {"type": "choice", "instructions": instructions,
+                "criteria": {criterion: None for criterion in criteria}}
+    if kind == "score":  # ordered levels
+        if not 2 <= len(criteria) <= MAX_SCORE_LEVELS:
+            raise ValueError(f"System One score questions take 2 to {MAX_SCORE_LEVELS} levels, got {len(criteria)}")
+        return {"type": "score", "instructions": instructions, "criteria": criteria}
+    raise ValueError(f"Unsupported System One question kind: {kind}")
+
+
+def render_typed_decision(example, question_id="decision", model=None):
+    """Render one canonical Jev row as a System One request with its question kind."""
     request = OrderedDict()
     if model is not None:
         request["model"] = model
     request["state"] = example["state"]
-    request["questions"] = {
-        question_id: {
-            "type": "choice",
-            "instructions": example["instructions"],
-            "criteria": {criterion: None for criterion in criteria},
-        }
-    }
+    request["questions"] = {question_id: _typed_question(example, example["instructions"])}
     return dict(request)
 
 
@@ -307,19 +319,13 @@ def render_typed_decision_group(examples, model=None):
         question_id = example.get("question_id", "decision")
         if question_id in questions:
             raise ValueError(f"Duplicate question id: {question_id}")
-        criteria = list(example["criteria"])
-        if len(criteria) != len(set(criteria)):
-            raise ValueError("System One choice criterion names must be unique")
         instructions = example["instructions"]
         if "target_index" in example:
             instructions += (
                 f" Target token at position {example['target_index']}: "
                 f"{example['target_token']}"
             )
-        questions[question_id] = {
-            "type": "choice", "instructions": instructions,
-            "criteria": {criterion: None for criterion in criteria},
-        }
+        questions[question_id] = _typed_question(example, instructions)
     request["questions"] = questions
     return dict(request)
 
diff --git a/src/tasksource/tasks.py b/src/tasksource/tasks.py
index 90c1989..6ba28a5 100755
--- a/src/tasksource/tasks.py
+++ b/src/tasksource/tasks.py
@@ -1664,13 +1664,15 @@ def _support_shift_name(shift):
 # test rows: drop them, and the corpora tasksource already has; multi-answer rows (13%) too.
 _INTENT_GRASP_LISTED = {"banking77", "clinc", "trec", "snips", "sci_cite", "mtop", "moral_stories"}
 
+def _intent_grasp_keep(x, split):
+    meta = x["metadata"]
+    return (len(x["answer_index"]) == 1 and 0 <= x["answer_index"][0] < len(x["options"])
+            and meta["original_task"] not in _INTENT_GRASP_LISTED
+            and (split == "test" or meta["original_split"] != "test"))
+
 def _intent_grasp(dataset):
-    def keep(x, split):
-        meta = x["metadata"]
-        return (len(x["answer_index"]) == 1 and x["answer_index"][0] < len(x["options"])
-                and meta["original_task"] not in _INTENT_GRASP_LISTED
-                and (split == "test" or meta["original_split"] != "test"))
-    return DatasetDict({split: rows.filter(keep, fn_kwargs={"split": split}) for split, rows in dataset.items()})
+    return DatasetDict({split: rows.filter(_intent_grasp_keep, fn_kwargs={"split": split})
+                        for split, rows in dataset.items()})
 
 intent_grasp = MultipleChoice(lambda x: f"{x['context']}\n\n{x['question']}",
     choices_list="options", labels=lambda x: x["answer_index"][0],
diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py
new file mode 100644
index 0000000..87eac66
--- /dev/null
+++ b/tests/test_build_provenance.py
@@ -0,0 +1,100 @@
+"""Regressions for build resume/provenance, typed rendering, and catalog lookup."""
+
+import argparse
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from tasksource import list_tasks, task_provenance
+from tasksource.access import load_preprocessing
+from tasksource.jev.recast import render_typed_decision, render_typed_decision_group
+from tasksource.tasks import _intent_grasp_keep
+from scripts.build_jev_dataset import build_fingerprint, read_completed, slug, source_provenance
+
+STATE = {"git_commit": "abc", "uncommitted_sha256": "0"}
+
+
+def _args(**overrides):
+    values = dict(max_rows=1000, max_rows_eval=100, noul_rate=0.05, score_rate=0.0, permutation_rate=0.05,
+                  prompt_rate=0.05, paired_format_rate=0.05, pack_rate=0.1, pack_max_tokens=4096,
+                  pack_tokenizer=None, pack_max_items=4, output=Path("a"), upload=False)
+    return argparse.Namespace(**{**values, **overrides})
+
+
+class ResumeTest(unittest.TestCase):
+    def test_fingerprint_tracks_code_and_shard_settings_only(self):
+        base = build_fingerprint(_args(), STATE)
+        self.assertNotEqual(base, build_fingerprint(_args(max_rows=30000), STATE))
+        self.assertNotEqual(base, build_fingerprint(_args(), {**STATE, "git_commit": "def"}))
+        self.assertNotEqual(base, build_fingerprint(_args(), {**STATE, "uncommitted_sha256": "1"}))
+        # where the output goes or whether it uploads does not change shard contents
+        self.assertEqual(base, build_fingerprint(_args(output=Path("b"), upload=True), STATE))
+
+    def test_shards_from_another_fingerprint_are_stale(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            data_dir, report = Path(tmp), Path(tmp) / "build-report.jsonl"
+            for task in ("old", "new", "legacy"):
+                (data_dir / f"train-{slug(task)}.parquet").touch()
+            records = [{"task": "old", "status": "ok", "rows": {"train": 1}, "fingerprint": "f1"},
+                       {"task": "new", "status": "ok", "rows": {"train": 1}, "fingerprint": "f2"},
+                       {"task": "legacy", "status": "ok", "rows": {"train": 1}}]  # built before fingerprints
+            report.write_text("".join(json.dumps(r) + "\n" for r in records))
+            self.assertEqual(read_completed(report, data_dir, "f2"), ({"new"}, {"old", "legacy"}))
+            self.assertEqual(read_completed(report, data_dir), {"old", "new", "legacy"})
+
+
+class ProvenanceTest(unittest.TestCase):
+    def test_requested_revisions_are_reported(self):
+        self.assertEqual(task_provenance("dream")["revision"], "refs/convert/parquet")
+        conll = task_provenance("conll2002/es", multilingual=True)
+        self.assertEqual(conll["data_file_revisions"], {"eriktks/conll2002": "refs/convert/parquet"})
+        self.assertEqual(source_provenance("graded/hatexplain")["revision"], "refs/convert/parquet")
+        self.assertNotIn("revision", task_provenance("glue/mnli"))
+
+
+class TypedRendererTest(unittest.TestCase):
+    ROW = {"state": "S", "instructions": "Q?", "criteria": ["low", "mid", "high"]}
+
+    def test_each_kind_renders_as_itself(self):
+        question = lambda kind: render_typed_decision({**self.ROW, "kind": kind})["questions"]["decision"]
+        self.assertEqual(question("choice")["criteria"], {"low": None, "mid": None, "high": None})
+        self.assertEqual((question("score")["type"], question("score")["criteria"]), ("score", ["low", "mid", "high"]))
+        self.assertEqual(question("noul"), {"type": "noul", "instructions": "Q?"})
+        self.assertEqual(render_typed_decision(self.ROW)["questions"]["decision"]["type"], "choice")  # default
+        with self.assertRaises(ValueError):
+            render_typed_decision({**self.ROW, "kind": "free_text"})
+        with self.assertRaises(ValueError):  # System One scores take 2 to 10 levels
+            render_typed_decision({**self.ROW, "kind": "score", "criteria": [str(i) for i in range(11)]})
+
+    def test_group_keeps_each_kind(self):
+        request = render_typed_decision_group([
+            {**self.ROW, "kind": "score", "question_id": "a"}, {**self.ROW, "kind": "choice", "question_id": "b"}])
+        self.assertEqual({qid: q["type"] for qid, q in request["questions"].items()}, {"a": "score", "b": "choice"})
+
+
+class CatalogApiTest(unittest.TestCase):
+    def test_intent_grasp_answer_bounds(self):
+        row = lambda index: {"answer_index": [index], "options": ["a", "b", "c"],
+                             "metadata": {"original_task": "x", "original_split": "train"}}
+        self.assertEqual([_intent_grasp_keep(row(i), "train") for i in (-1, 0, 2, 3)], [False, True, True, False])
+
+    def test_list_tasks_accepts_lists_and_returns_copies(self):
+        self.assertFalse(list_tasks(excluded=["glue/"]).id.str.contains("glue/").any())
+        full = len(list_tasks())
+        list_tasks().drop(list_tasks().index[:5], inplace=True)
+        edited = list_tasks()
+        edited["extra"] = 1
+        self.assertEqual(len(list_tasks()), full)
+        self.assertNotIn("extra", list_tasks().columns)
+
+    def test_lookup_errors(self):
+        with self.assertRaises(KeyError):
+            load_preprocessing(id="no-such-task")
+        with self.assertRaises(ValueError):  # a dataset name alone matches every glue config
+            load_preprocessing(dataset_name="nyu-mll/glue")
+        self.assertEqual(load_preprocessing(id="glue/rte").config_name, "rte")
+
+
+if __name__ == "__main__":
+    unittest.main()

From bb9241bf66b3acb29bdb9aa57023580a2dda5e2d Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 25 Sep 2026 08:57:19 +0000
Subject: [PATCH 2/5] WildGuardMix split by prompt; BeaverTails one row per
 pair with its majority, plus its vote share as a graded family

- the WildGuardMix mirror has one train split with two responses per prompt, so the random
  validation/test split put 90% of evaluation prompts in train; all three tasks now split by
  a stable hash of the prompt (split_by_group), and prompt_harm keeps one row per prompt
- BeaverTails annotates each (prompt, response) three times as separate rows (27.6% of pairs
  split): the task keeps one row per pair with the majority judgment, which also ends the
  89% validation-in-train overlap; graded/beavertails gives the share of annotators who
  judged the response unsafe (0, 1/3, 2/3, 1)
- ORIGINALS: the mirror's original is allenai/wildguardmix

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EVp7VRXPWTGT6HsHHWJghc
---
 src/tasksource/jev/graded.py         | 17 +++++++++++-
 src/tasksource/metadata/originals.py |  2 ++
 src/tasksource/tasks.py              | 40 +++++++++++++++++++++++++---
 tests/test_build_provenance.py       | 25 +++++++++++++++++
 4 files changed, 80 insertions(+), 4 deletions(-)

diff --git a/src/tasksource/jev/graded.py b/src/tasksource/jev/graded.py
index d6234ed..c4eb74b 100644
--- a/src/tasksource/jev/graded.py
+++ b/src/tasksource/jev/graded.py
@@ -10,7 +10,7 @@
 import math
 from dataclasses import dataclass
 
-from datasets import DatasetDict, load_dataset
+from datasets import Dataset, DatasetDict, load_dataset
 
 from ..preprocess import fix_splits, sample_dataset
 from ..tasks import render_dialogue, render_helpsteer_prompt
@@ -67,6 +67,7 @@ class Family:
     prepare: object = None  # row map adding the state columns
     dedupe: str = None  # column whose repeats are dropped (one row per annotator upstream)
     load_kwargs: dict = None  # e.g. the Hub parquet export of a script-only dataset
+    pre_process: object = None  # DatasetDict -> DatasetDict, e.g. one row per item from per-annotator rows
 
 
 HELPSTEER = dict(
@@ -134,6 +135,15 @@ def _dynasent_votes(example):
     return {"votes": [len(annotators.get(label, [])) for label in DYNASENT]}
 
 
+def _beavertails_votes(dataset):
+    """One row per (prompt, response) with the share of its three annotators who judged it unsafe."""
+    def votes(rows):
+        frame = rows.to_pandas()
+        frame["unsafe_share"] = frame.groupby(["prompt", "response"]).is_safe.transform(lambda safe: (~safe).sum() / len(safe))
+        return Dataset.from_pandas(frame.drop_duplicates(["prompt", "response"]), preserve_index=False)
+    return DatasetDict(train=votes(dataset["330k_train"]), test=votes(dataset["330k_test"]))
+
+
 def _share_of_yes(example):
     return {"share": example["soft_label"][1]}
 
@@ -198,6 +208,9 @@ def _lewidi_noul(config, state, question, prepare=_share_of_yes):
         disagreement=noul("disagreement_rate", f"How much would annotators disagree about {topic}, "
                                                "from 0 (all agree) to 1 (maximal disagreement)?")),
         dedupe="text") for name, topic in DISAGREEMENT.items()},
+    "beavertails": Family("PKU-Alignment/BeaverTails", {"Prompt": "prompt", "Response": "response"}, dict(
+        unsafe=noul("unsafe_share", "What fraction of annotators judged the assistant response unsafe?")),
+        pre_process=_beavertails_votes),
     "unli": Family("Zhengping/UNLI", PREMISE, dict(
         probability=noul("label", "How likely is the hypothesis to be true, given the premise?"))),
     **{f"dynasent_{r}": Family("dynabench/dynasent", {"Sentence": "sentence"}, dict(
@@ -270,6 +283,8 @@ def load_family(name, max_rows=None, max_rows_eval=None, revision=None):
     family = FAMILIES[name]
     kwargs = {**(family.load_kwargs or {}), **({"revision": revision} if revision else {})}
     dataset = DatasetDict(load_dataset(family.dataset, family.config, **kwargs))
+    if family.pre_process:
+        dataset = family.pre_process(dataset)
     if family.dedupe:
         dataset = DatasetDict({split: rows.select(
             rows.to_pandas().drop_duplicates(family.dedupe).index.tolist()) for split, rows in dataset.items()})
diff --git a/src/tasksource/metadata/originals.py b/src/tasksource/metadata/originals.py
index c0edfcc..841125f 100644
--- a/src/tasksource/metadata/originals.py
+++ b/src/tasksource/metadata/originals.py
@@ -11,6 +11,8 @@
 """
 
 ORIGINALS = {
+    # third-party mirrors
+    "bogdanminko/wildguardmix-cleaned": ["allenai/wildguardmix"],
     # re-uploaded under tasksource/
     "tasksource/blog_authorship_corpus": ["barilan/blog_authorship_corpus"],
     "tasksource/chaos-mnli-ambiguity": [],  # ChaosNLI is only on GitHub/Dropbox
diff --git a/src/tasksource/tasks.py b/src/tasksource/tasks.py
index 6ba28a5..d104f50 100755
--- a/src/tasksource/tasks.py
+++ b/src/tasksource/tasks.py
@@ -1,6 +1,7 @@
 from .preprocess import cat, get, regen, name, constant, Classification, TokenClassification, MultipleChoice
 from .metadata import udep_en_configs
-from datasets import get_dataset_config_names, Sequence, ClassLabel, Dataset, DatasetDict, Features, Value
+from datasets import get_dataset_config_names, Sequence, ClassLabel, Dataset, DatasetDict, Features, Value, concatenate_datasets
+import hashlib
 import html
 from collections import Counter
 import random
@@ -1623,25 +1624,58 @@ def _support_shift_name(shift):
     dataset_name="kontext-security/ShellRisk-Bench",
     question="Is this shell command risky?")
 
+def split_by_group(dataset, key, validation=0.05, test=0.05):
+    """Re-split every row by a stable hash of ``key(row)``, so a group (e.g. all rows
+    sharing a prompt) never straddles train and evaluation."""
+    def split_of(row):
+        fraction = int(hashlib.sha256(key(row).encode("utf-8")).hexdigest()[:8], 16) / 16 ** 8
+        return "test" if fraction < test else "validation" if fraction < test + validation else "train"
+    rows = concatenate_datasets(list(dataset.values()))
+    splits = rows.map(lambda row: {"_split": split_of(row)})["_split"]
+    return DatasetDict({name: rows.select([i for i, s in enumerate(splits) if s == name])
+                        for name in ("train", "validation", "test")})
+
+# The mirror has one train split with two responses per prompt: split by prompt, so no
+# evaluation prompt is seen in training (a random split put 90% of them there).
+def _wildguardmix(dataset, one_row_per_prompt=False):
+    dataset = split_by_group(dataset, lambda row: row["prompt"])
+    if one_row_per_prompt:  # prompt labels never conflict across a prompt's rows
+        dataset = DatasetDict({split: rows.select(rows.to_pandas().drop_duplicates("prompt").index.tolist())
+                               for split, rows in dataset.items()})
+    return dataset
+
 wildguardmix__prompt_harm = Classification(
     "prompt", labels="prompt_harm_label",
     dataset_name="bogdanminko/wildguardmix-cleaned",
+    pre_process=lambda ds: _wildguardmix(ds, one_row_per_prompt=True),
     question="Is this user prompt harmful?")
 
 wildguardmix__response_harm = Classification(
     "prompt", "response", labels="response_harm_label",
-    dataset_name="bogdanminko/wildguardmix-cleaned",
+    dataset_name="bogdanminko/wildguardmix-cleaned", pre_process=_wildguardmix,
     question="Is the assistant response harmful?")
 
 wildguardmix__response_refusal = Classification(
     "prompt", "response", labels="response_refusal_label",
-    dataset_name="bogdanminko/wildguardmix-cleaned",
+    dataset_name="bogdanminko/wildguardmix-cleaned", pre_process=_wildguardmix,
     question="Does the assistant response refuse the request?")
 
+# BeaverTails annotates each (prompt, response) three times, one row per annotation;
+# keep one row per pair with the majority judgment (the vote share is graded/beavertails).
+def _beavertails_majority(dataset):
+    def majority(rows):
+        frame = rows.to_pandas()
+        frame["safe_votes"] = frame.groupby(["prompt", "response"]).is_safe.transform("mean")
+        frame = frame.drop_duplicates(["prompt", "response"])
+        frame["is_safe"] = frame.safe_votes > 0.5  # three votes: never a tie
+        return Dataset.from_pandas(frame.drop(columns="safe_votes"), preserve_index=False)
+    return DatasetDict({split: majority(rows) for split, rows in dataset.items()})
+
 beavertails_safety = Classification(
     "prompt", "response", labels=name("is_safe", ["unsafe", "safe"]),
     dataset_name="PKU-Alignment/BeaverTails",
     splits=["330k_train", None, "330k_test"],
+    pre_process=_beavertails_majority,
     question="Is the assistant response safe?")
 
 
diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py
index 87eac66..45941b7 100644
--- a/tests/test_build_provenance.py
+++ b/tests/test_build_provenance.py
@@ -96,5 +96,30 @@ def test_lookup_errors(self):
         self.assertEqual(load_preprocessing(id="glue/rte").config_name, "rte")
 
 
+class SafetySplitTest(unittest.TestCase):
+    def test_grouped_split_keeps_prompts_on_one_side(self):
+        from datasets import Dataset, DatasetDict
+        from tasksource.tasks import _wildguardmix
+        rows = Dataset.from_dict({"prompt": [f"p{i // 2}" for i in range(4000)], "response": ["r"] * 4000,
+                                  "prompt_harm_label": ["harmful"] * 4000})
+        split = _wildguardmix(DatasetDict(train=rows))
+        prompts = {name: set(part["prompt"]) for name, part in split.items()}
+        self.assertFalse(prompts["train"] & (prompts["validation"] | prompts["test"]))
+        self.assertTrue(all(len(part) for part in split.values()))
+        one = _wildguardmix(DatasetDict(train=rows), one_row_per_prompt=True)
+        self.assertEqual(sum(map(len, one.values())), 2000)
+
+    def test_beavertails_votes(self):
+        from datasets import Dataset, DatasetDict
+        from tasksource.tasks import _beavertails_majority
+        from tasksource.jev.graded import _beavertails_votes
+        rows = Dataset.from_dict({"prompt": ["p"] * 3 + ["q"] * 3, "response": ["r"] * 3 + ["s"] * 3,
+                                  "category": [{}] * 6, "is_safe": [True, True, False, False, False, True]})
+        majority = _beavertails_majority(DatasetDict({"330k_train": rows}))["330k_train"]
+        self.assertEqual(sorted(zip(majority["prompt"], majority["is_safe"])), [("p", True), ("q", False)])
+        votes = _beavertails_votes(DatasetDict({"330k_train": rows, "330k_test": rows}))["train"]
+        self.assertEqual(sorted(zip(votes["prompt"], votes["unsafe_share"])), [("p", 1 / 3), ("q", 2 / 3)])
+
+
 if __name__ == "__main__":
     unittest.main()

From dc04ccb0a235b49d5e1da809bf1381a0d6cb092c Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 25 Sep 2026 09:04:19 +0000
Subject: [PATCH 3/5] load_task: multiple-choice options no longer put the gold
 answer first

Plain load_task put the gold answer in slot 0 for every row of multiple-choice tasks
built from option lists (hellaswag, ARC, IntentGrasp: the gold went first to allow
truncation) and of fixed-column tasks whose annotation lists the gold first
(labels=constant(0): sciq, hh-rlhf, UltraFeedback, prm800k, the DPO pairs, ...). The
kept options are now permuted, deterministically from the options' text; rows whose
options refer to each other by position keep the source order. Choice columns are also
read in numeric order (choice10 after choice2). The Jev and instruct recasts already
shuffled and are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EVp7VRXPWTGT6HsHHWJghc
---
 src/tasksource/preprocess.py | 37 +++++++++++++++++++++---------------
 tests/test_jev_derived.py    | 10 +++++++++-
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/src/tasksource/preprocess.py b/src/tasksource/preprocess.py
index c166a4b..07559d3 100755
--- a/src/tasksource/preprocess.py
+++ b/src/tasksource/preprocess.py
@@ -3,6 +3,7 @@
 from dataclasses import dataclass, field
 from typing import Union
 import itertools
+import re
 import funcy as fc
 import exrex 
 import magicattr 
@@ -217,15 +218,23 @@ def ordered_sample_choices(x, n_options=None):
             x[f'choice{i}']=o
         return x
 
+    @staticmethod
+    def shuffled_with_gold(choices, label, n_options=None):
+        """Keep the gold option and the first negatives in source order, then permute them so the
+        gold slot carries no signal: deterministic, seeded by the options' text. Options that refer
+        to each other by position ("all of the above") keep the source order."""
+        from .jev.options import choice_permutation  # at call time: jev imports this module
+        kept_indices = sorted([label, *[i for i in range(len(choices)) if i != label][:(n_options or len(choices)) - 1]])
+        kept, gold = [choices[i] for i in kept_indices], kept_indices.index(label)
+        order = choice_permutation(kept, "\x1f".join(map(str, kept)))
+        if order is None:  # options refer to each other by position: keep the source order
+            return kept, gold
+        return [kept[i] for i in order], order.index(gold)
+
     @staticmethod
     def flatten_choice_list(x, n_options=None):
-        n_neg = n_options-1 if n_options else None
-        choices = x['choices_list']
-        label=x['labels']
-        neg = choices[:label] + choices[label+1:]
-        pos = choices[label]
-        x['labels']=0
-        x['choices_list']=[pos]+neg[:n_neg]
+        x['choices_list'], x['labels'] = MultipleChoiceFields.shuffled_with_gold(
+            x['choices_list'], x['labels'], n_options)
         for i,o in enumerate(x['choices_list']):
             x[f'choice{i}']=o
         del x['choices_list']
@@ -233,15 +242,13 @@ def flatten_choice_list(x, n_options=None):
 
     @staticmethod
     def sample_choices(x, n_options=None):
-        choices = [x[c] for c in x if 'choice' in c]
-        if not MAX_MC_OPTIONS or len(choices)<=n_options:
+        names = sorted((c for c in x if re.fullmatch(r'choice\d+', c)), key=lambda c: int(c[6:]))  # choice10 after choice2
+        choices = [x[c] for c in names]
+        # also when nothing is truncated: many annotations put the gold answer first (labels=constant(0))
+        if not 0 <= x['labels'] < len(choices):
             return x
-        n_neg = n_options-1 if n_options else None
-        label=x['labels']
-        neg = choices[:label] + choices[label+1:]
-        pos = choices[label]
-        x['labels']=0
-        choices_list=[pos]+neg[:n_neg]
+        choices_list, x['labels'] = MultipleChoiceFields.shuffled_with_gold(
+            choices, x['labels'], n_options if MAX_MC_OPTIONS else None)
         for c in list(x):
             if 'choice' in c:
                 del x[c]
diff --git a/tests/test_jev_derived.py b/tests/test_jev_derived.py
index bca6fea..d0cd17f 100644
--- a/tests/test_jev_derived.py
+++ b/tests/test_jev_derived.py
@@ -95,7 +95,15 @@ def test_jev_preprocessing_keeps_order_and_all_options(self):
         })})
         preprocessing = MultipleChoice("q", choices_list="opts", labels="label")
         legacy = preprocessing(DatasetDict(source), gold_first=True)["train"]
-        self.assertEqual(set(legacy["labels"]), {0})
+        # truncated to the shortest row's option count, gold kept, then shuffled: gold is no longer always first
+        for row in legacy:
+            self.assertEqual(row[f"choice{row['labels']}"], {"a": "y", "b": "q"}[row["inputs"]])
+        # over many distinct option sets, the gold slot is spread out
+        many = DatasetDict({"train": Dataset.from_dict({
+            "q": [f"q{i}" for i in range(400)], "opts": [[f"{i}-{c}" for c in "abcdef"] for i in range(400)],
+            "label": [i % 6 for i in range(400)]})})
+        slots = Counter(preprocessing(many, gold_first=True)["train"]["labels"])
+        self.assertTrue(all(count > 40 for count in slots.values()) and len(slots) == 4, slots)
         jev = preprocessing(DatasetDict(source), gold_first=False, max_options=None)["train"]
         jev = jev.sort("inputs")
         self.assertEqual(jev["labels"][0], 2)

From 02267a6cf8ffe95f675c987dcbe16192e0411101 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 25 Sep 2026 09:09:19 +0000
Subject: [PATCH 4/5] Add privacy-200k-Mistral-Large-3: privacy sensitivity
 rated 1-5, as an ordinal task

Mistral Large 3 ratings over ten domains (Loiseau et al., 2026, arXiv:2603.29497). The
question and level names follow the annotation prompt's scale (identifiers, personal
information, should not be made public); ordinal=True puts the levels in scale order.
The source splits are domains: they are merged (a domain column is kept) and split
90/5/5 by text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EVp7VRXPWTGT6HsHHWJghc
---
 src/tasksource/tasks.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/tasksource/tasks.py b/src/tasksource/tasks.py
index d104f50..f69a0c8 100755
--- a/src/tasksource/tasks.py
+++ b/src/tasksource/tasks.py
@@ -1679,6 +1679,23 @@ def majority(rows):
     question="Is the assistant response safe?")
 
 
+# Privacy sensitivity rated 1-5 by Mistral Large 3 (a teacher LLM, not people) over ten domains
+# (Loiseau et al., 2026, arXiv:2603.29497); label names follow the annotation prompt's scale.
+# The source splits are domains: merge them and split by text, so no text straddles splits.
+def _privacy_200k(dataset):
+    rows = concatenate_datasets([part.add_column("domain", [name] * len(part)) for name, part in dataset.items()])
+    return split_by_group(DatasetDict(train=rows), lambda row: " ".join(row["text"].split()).lower())
+
+privacy_200k = Classification("text", labels="label", ordinal=True, pre_process=_privacy_200k,
+    question="How private or sensitive is this text, i.e. how much personal information or direct or indirect "
+             "identifiers does it contain?",
+    label_values={1: "not private: no direct or indirect identifiers",
+                  2: "mostly not private: some indirect identifiers at most",
+                  3: "somewhat private: some identifiers, somewhat personal",
+                  4: "very private: several identifiers, clearly personal",
+                  5: "extremely private: highly sensitive, should not be made public"},
+    dataset_name="gabrielloiseau/privacy-200k-Mistral-Large-3", task_id="privacy-200k-Mistral-Large-3")
+
 toxic_chat__toxicity = Classification(
     "user_input", labels=name("toxicity", ["not toxic", "toxic"]), question="Is this user prompt toxic?",
     dataset_name="lmsys/toxic-chat", config_name="toxicchat0124",

From d6a7f1a13d5820f3ed003fc790a92c9946f3f4ff Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Fri, 25 Sep 2026 09:09:39 +0000
Subject: [PATCH 5/5] Regenerate task catalogs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EVp7VRXPWTGT6HsHHWJghc
---
 tasks.md | 991 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 496 insertions(+), 495 deletions(-)

diff --git a/tasks.md b/tasks.md
index ce92ee4..771148a 100644
--- a/tasks.md
+++ b/tasks.md
@@ -1,498 +1,499 @@
-494 English tasks. Load one with `load_task(id)`; the annotations are in [tasks.py](src/tasksource/tasks.py), and tasks kept out on purpose are in [parked.py](src/tasksource/parked.py).
+495 English tasks. Load one with `load_task(id)`; the annotations are in [tasks.py](src/tasksource/tasks.py), and tasks kept out on purpose are in [parked.py](src/tasksource/parked.py).
 
 | # | id | type | dataset | question |
 |--:|---|---|---|:-:|
-| 1 | [glue/mnli](src/tasksource/tasks.py#L28) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 2 | [glue/qnli](src/tasksource/tasks.py#L29) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 3 | [glue/rte](src/tasksource/tasks.py#L30) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 4 | [glue/wnli](src/tasksource/tasks.py#L31) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 5 | [glue/mrpc](src/tasksource/tasks.py#L33) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 6 | [glue/qqp](src/tasksource/tasks.py#L34) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 7 | [glue/stsb](src/tasksource/tasks.py#L35) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) | ✓ |
-| 8 | [super_glue/boolq](src/tasksource/tasks.py#L38) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 9 | [super_glue/boolq_passage](src/tasksource/tasks.py#L39) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 10 | [super_glue/cb](src/tasksource/tasks.py#L41) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 11 | [super_glue/multirc](src/tasksource/tasks.py#L42) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 12 | [super_glue/wic](src/tasksource/tasks.py#L47) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 13 | [super_glue/axg](src/tasksource/tasks.py#L52) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 14 | [anli/a1](src/tasksource/tasks.py#L55) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
-| 15 | [anli/a2](src/tasksource/tasks.py#L56) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
-| 16 | [anli/a3](src/tasksource/tasks.py#L57) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
-| 17 | [babi_nli/basic-deduction](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 18 | [babi_nli/conjunction](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 19 | [babi_nli/compound-coreference](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 20 | [babi_nli/basic-induction](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 21 | [babi_nli/lists-sets](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 22 | [babi_nli/basic-coreference](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 23 | [babi_nli/path-finding](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 24 | [babi_nli/positional-reasoning](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 25 | [babi_nli/single-supporting-fact](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 26 | [babi_nli/simple-negation](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 27 | [babi_nli/counting](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 28 | [babi_nli/size-reasoning](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 29 | [babi_nli/three-arg-relations](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 30 | [babi_nli/three-supporting-facts](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 31 | [babi_nli/time-reasoning](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 32 | [babi_nli/two-arg-relations](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 33 | [babi_nli/two-supporting-facts](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 34 | [babi_nli/yes-no-questions](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 35 | [babi_nli/indefinite-knowledge](src/tasksource/tasks.py#L60) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
-| 36 | [sick/label](src/tasksource/tasks.py#L66) | Classification | [tasksource/sick](https://hf.co/datasets/tasksource/sick) |  |
-| 37 | [sick/relatedness](src/tasksource/tasks.py#L67) | Classification | [tasksource/sick](https://hf.co/datasets/tasksource/sick) | ✓ |
-| 38 | [snli](src/tasksource/tasks.py#L122) | Classification | [stanfordnlp/snli](https://hf.co/datasets/stanfordnlp/snli) |  |
-| 39 | [scitail/snli_format](src/tasksource/tasks.py#L125) | Classification | [allenai/scitail](https://hf.co/datasets/allenai/scitail) |  |
-| 40 | [hans](src/tasksource/tasks.py#L127) | Classification | [tasksource/hans](https://hf.co/datasets/tasksource/hans) |  |
-| 41 | [WANLI](src/tasksource/tasks.py#L130) | Classification | [alisawuffles/WANLI](https://hf.co/datasets/alisawuffles/WANLI) |  |
-| 42 | [recast/recast_megaveridicality](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 43 | [recast/recast_sentiment](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 44 | [recast/recast_ner](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 45 | [recast/recast_verbcorner](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 46 | [recast/recast_verbnet](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 47 | [recast/recast_factuality](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 48 | [recast/recast_puns](src/tasksource/tasks.py#L132) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
-| 49 | [probability_words_nli/reasoning_1hop](src/tasksource/tasks.py#L137) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
-| 50 | [probability_words_nli/reasoning_2hop](src/tasksource/tasks.py#L137) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
-| 51 | [probability_words_nli/usnli](src/tasksource/tasks.py#L137) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
-| 52 | [nan-nli](src/tasksource/tasks.py#L141) | Classification | [joey234/nan-nli](https://hf.co/datasets/joey234/nan-nli) |  |
-| 53 | [nli_fever](src/tasksource/tasks.py#L143) | Classification | [pietrolesci/nli_fever](https://hf.co/datasets/pietrolesci/nli_fever) |  |
-| 54 | [breaking_nli](src/tasksource/tasks.py#L146) | Classification | [pietrolesci/breaking_nli](https://hf.co/datasets/pietrolesci/breaking_nli) |  |
-| 55 | [conj_nli](src/tasksource/tasks.py#L150) | Classification | [pietrolesci/conj_nli](https://hf.co/datasets/pietrolesci/conj_nli) |  |
-| 56 | [fracas](src/tasksource/tasks.py#L154) | Classification | [pietrolesci/fracas](https://hf.co/datasets/pietrolesci/fracas) |  |
-| 57 | [dialogue_nli](src/tasksource/tasks.py#L157) | Classification | [pietrolesci/dialogue_nli](https://hf.co/datasets/pietrolesci/dialogue_nli) |  |
-| 58 | [mpe](src/tasksource/tasks.py#L160) | Classification | [pietrolesci/mpe](https://hf.co/datasets/pietrolesci/mpe) |  |
-| 59 | [dnc](src/tasksource/tasks.py#L164) | Classification | [pietrolesci/dnc](https://hf.co/datasets/pietrolesci/dnc) |  |
-| 60 | [recast_white/fnplus](src/tasksource/tasks.py#L168) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
-| 61 | [recast_white/sprl](src/tasksource/tasks.py#L171) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
-| 62 | [recast_white/dpr](src/tasksource/tasks.py#L174) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
-| 63 | [joci](src/tasksource/tasks.py#L178) | Classification | [pietrolesci/joci](https://hf.co/datasets/pietrolesci/joci) |  |
-| 64 | [robust_nli/IS_CS](src/tasksource/tasks.py#L184) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 65 | [robust_nli/LI_LI](src/tasksource/tasks.py#L186) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 66 | [robust_nli/ST_WO](src/tasksource/tasks.py#L188) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 67 | [robust_nli/PI_SP](src/tasksource/tasks.py#L190) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 68 | [robust_nli/PI_CD](src/tasksource/tasks.py#L192) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 69 | [robust_nli/ST_SE](src/tasksource/tasks.py#L194) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 70 | [robust_nli/ST_NE](src/tasksource/tasks.py#L196) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 71 | [robust_nli/ST_LM](src/tasksource/tasks.py#L198) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
-| 72 | [robust_nli_is_sd](src/tasksource/tasks.py#L200) | Classification | [pietrolesci/robust_nli_is_sd](https://hf.co/datasets/pietrolesci/robust_nli_is_sd) |  |
-| 73 | [robust_nli_li_ts](src/tasksource/tasks.py#L203) | Classification | [pietrolesci/robust_nli_li_ts](https://hf.co/datasets/pietrolesci/robust_nli_li_ts) |  |
-| 74 | [gen_debiased_nli/snli_seq_z](src/tasksource/tasks.py#L207) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 75 | [gen_debiased_nli/snli_z_aug](src/tasksource/tasks.py#L209) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 76 | [gen_debiased_nli/snli_par_z](src/tasksource/tasks.py#L211) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 77 | [gen_debiased_nli/mnli_par_z](src/tasksource/tasks.py#L213) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 78 | [gen_debiased_nli/mnli_z_aug](src/tasksource/tasks.py#L215) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 79 | [gen_debiased_nli/mnli_seq_z](src/tasksource/tasks.py#L217) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
-| 80 | [add_one_rte](src/tasksource/tasks.py#L220) | Classification | [pietrolesci/add_one_rte](https://hf.co/datasets/pietrolesci/add_one_rte) |  |
-| 81 | [hlgd](src/tasksource/tasks.py#L224) | Classification | [tasksource/hlgd](https://hf.co/datasets/tasksource/hlgd) |  |
-| 82 | [paws/labeled_final](src/tasksource/tasks.py#L226) | Classification | [google-research-datasets/paws](https://hf.co/datasets/google-research-datasets/paws) |  |
-| 83 | [paws/labeled_swap](src/tasksource/tasks.py#L227) | Classification | [google-research-datasets/paws](https://hf.co/datasets/google-research-datasets/paws) |  |
-| 84 | [medical_questions_pairs](src/tasksource/tasks.py#L229) | Classification | [curaihealth/medical_questions_pairs](https://hf.co/datasets/curaihealth/medical_questions_pairs) |  |
-| 85 | [conll2003/pos_tags](src/tasksource/tasks.py#L234) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
-| 86 | [conll2003/chunk_tags](src/tasksource/tasks.py#L235) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
-| 87 | [conll2003/ner_tags](src/tasksource/tasks.py#L236) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
-| 88 | [fig-qa](src/tasksource/tasks.py#L242) | MultipleChoice | [nightingal3/fig-qa](https://hf.co/datasets/nightingal3/fig-qa) |  |
-| 89 | [cos_e/v1.0](src/tasksource/tasks.py#L251) | MultipleChoice | [Salesforce/cos_e](https://hf.co/datasets/Salesforce/cos_e) |  |
-| 90 | [cosmos_qa](src/tasksource/tasks.py#L256) | MultipleChoice | [Samsoup/cosmos_qa](https://hf.co/datasets/Samsoup/cosmos_qa) |  |
-| 91 | [dream](src/tasksource/tasks.py#L259) | MultipleChoice | [dataset-org/dream](https://hf.co/datasets/dataset-org/dream) |  |
-| 92 | [openbookqa](src/tasksource/tasks.py#L266) | MultipleChoice | [allenai/openbookqa](https://hf.co/datasets/allenai/openbookqa) |  |
-| 93 | [qasc](src/tasksource/tasks.py#L272) | MultipleChoice | [allenai/qasc](https://hf.co/datasets/allenai/qasc) |  |
-| 94 | [quartz](src/tasksource/tasks.py#L280) | MultipleChoice | [allenai/quartz](https://hf.co/datasets/allenai/quartz) |  |
-| 95 | [quail](src/tasksource/tasks.py#L285) | MultipleChoice | [textmachinelab/quail](https://hf.co/datasets/textmachinelab/quail) |  |
-| 96 | [head_qa/en](src/tasksource/tasks.py#L291) | MultipleChoice | [EleutherAI/headqa](https://hf.co/datasets/EleutherAI/headqa) |  |
-| 97 | [sciq](src/tasksource/tasks.py#L299) | MultipleChoice | [allenai/sciq](https://hf.co/datasets/allenai/sciq) |  |
-| 98 | [social_i_qa](src/tasksource/tasks.py#L304) | MultipleChoice | [tasksource/social_i_qa](https://hf.co/datasets/tasksource/social_i_qa) |  |
-| 99 | [wiki_hop/original](src/tasksource/tasks.py#L310) | MultipleChoice | [MoE-UNC/wikihop](https://hf.co/datasets/MoE-UNC/wikihop) |  |
-| 100 | [wiqa](src/tasksource/tasks.py#L317) | MultipleChoice | [tasksource/wiqa](https://hf.co/datasets/tasksource/wiqa) |  |
-| 101 | [piqa](src/tasksource/tasks.py#L322) | MultipleChoice | [baber/piqa](https://hf.co/datasets/baber/piqa) |  |
-| 102 | [hellaswag](src/tasksource/tasks.py#L330) | MultipleChoice | [Rowan/hellaswag](https://hf.co/datasets/Rowan/hellaswag) |  |
-| 103 | [super_glue/copa](src/tasksource/tasks.py#L339) | MultipleChoice | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
-| 104 | [balanced-copa](src/tasksource/tasks.py#L341) | MultipleChoice | [pkavumba/balanced-copa](https://hf.co/datasets/pkavumba/balanced-copa) |  |
-| 105 | [e-CARE](src/tasksource/tasks.py#L344) | MultipleChoice | [12ml/e-CARE](https://hf.co/datasets/12ml/e-CARE) |  |
-| 106 | [art](src/tasksource/tasks.py#L347) | MultipleChoice | [allenai/art](https://hf.co/datasets/allenai/art) | ✓ |
-| 107 | [winogrande/winogrande_xl](src/tasksource/tasks.py#L355) | MultipleChoice | [allenai/winogrande](https://hf.co/datasets/allenai/winogrande) |  |
-| 108 | [codah/codah](src/tasksource/tasks.py#L358) | MultipleChoice | [jaredfern/codah](https://hf.co/datasets/jaredfern/codah) |  |
-| 109 | [ai2_arc/ARC-Easy/challenge](src/tasksource/tasks.py#L360) | MultipleChoice | [allenai/ai2_arc](https://hf.co/datasets/allenai/ai2_arc) |  |
-| 110 | [ai2_arc/ARC-Challenge/challenge](src/tasksource/tasks.py#L360) | MultipleChoice | [allenai/ai2_arc](https://hf.co/datasets/allenai/ai2_arc) |  |
-| 111 | [definite_pronoun_resolution](src/tasksource/tasks.py#L365) | MultipleChoice | [community-datasets/definite_pronoun_resolution](https://hf.co/datasets/community-datasets/definite_pronoun_resolution) |  |
-| 112 | [swag/regular](src/tasksource/tasks.py#L371) | MultipleChoice | [allenai/swag](https://hf.co/datasets/allenai/swag) |  |
-| 113 | [math_qa](src/tasksource/tasks.py#L377) | MultipleChoice | [tasksource/math_qa](https://hf.co/datasets/tasksource/math_qa) |  |
-| 114 | [glue/cola](src/tasksource/tasks.py#L386) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 115 | [glue/sst2](src/tasksource/tasks.py#L387) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
-| 116 | [utilitarianism](src/tasksource/tasks.py#L401) | Classification | csv |  |
-| 117 | [amazon_counterfactual/en](src/tasksource/tasks.py#L409) | Classification | [mteb/amazon_counterfactual](https://hf.co/datasets/mteb/amazon_counterfactual) |  |
-| 118 | [insincere-questions](src/tasksource/tasks.py#L414) | Classification | [SetFit/insincere-questions](https://hf.co/datasets/SetFit/insincere-questions) |  |
-| 119 | [toxic_conversations](src/tasksource/tasks.py#L418) | Classification | [SetFit/toxic_conversations](https://hf.co/datasets/SetFit/toxic_conversations) |  |
-| 120 | [TuringBench](src/tasksource/tasks.py#L422) | Classification | csv |  |
-| 121 | [trec](src/tasksource/tasks.py#L431) | Classification | [tasksource/trec](https://hf.co/datasets/tasksource/trec) |  |
-| 122 | [vitaminc](src/tasksource/tasks.py#L434) | Classification | [tals/vitaminc](https://hf.co/datasets/tals/vitaminc) |  |
-| 123 | [hope_edi/english](src/tasksource/tasks.py#L436) | Classification | csv |  |
-| 124 | [rumoureval_2019/RumourEval2019](src/tasksource/tasks.py#L450) | Classification | csv |  |
-| 125 | [ethos/binary](src/tasksource/tasks.py#L464) | Classification | [SetFit/ethos_binary](https://hf.co/datasets/SetFit/ethos_binary) |  |
-| 126 | [ethos/multilabel](src/tasksource/tasks.py#L488) | Classification | [tasksource/ethos](https://hf.co/datasets/tasksource/ethos) |  |
-| 127 | [tweet_eval/emoji](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 128 | [tweet_eval/emotion](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 129 | [tweet_eval/hate](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 130 | [tweet_eval/sentiment](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 131 | [tweet_eval/irony](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 132 | [tweet_eval/offensive](src/tasksource/tasks.py#L491) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
-| 133 | [tweet_eval/stance_abortion](src/tasksource/tasks.py#L506) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
-| 134 | [tweet_eval/stance_atheism](src/tasksource/tasks.py#L507) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
-| 135 | [tweet_eval/stance_climate](src/tasksource/tasks.py#L508) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
-| 136 | [tweet_eval/stance_feminist](src/tasksource/tasks.py#L509) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
-| 137 | [tweet_eval/stance_hillary](src/tasksource/tasks.py#L510) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
-| 138 | [discovery/discovery](src/tasksource/tasks.py#L513) | Classification | [sileod/discovery](https://hf.co/datasets/sileod/discovery) |  |
-| 139 | [pragmeval/mrda](src/tasksource/tasks.py#L515) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 140 | [pragmeval/verifiability](src/tasksource/tasks.py#L515) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 141 | [pragmeval/switchboard](src/tasksource/tasks.py#L515) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 142 | [pragmeval/gum](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 143 | [pragmeval/emergent](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 144 | [pragmeval/persuasiveness-premisetype](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 145 | [pragmeval/stac](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 146 | [pragmeval/persuasiveness-claimtype](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 147 | [pragmeval/pdtb](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 148 | [pragmeval/sarcasm](src/tasksource/tasks.py#L519) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 149 | [pragmeval/emobank-arousal](src/tasksource/tasks.py#L528) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 150 | [pragmeval/emobank-dominance](src/tasksource/tasks.py#L529) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 151 | [pragmeval/emobank-valence](src/tasksource/tasks.py#L530) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 152 | [pragmeval/squinky-formality](src/tasksource/tasks.py#L531) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 153 | [pragmeval/squinky-implicature](src/tasksource/tasks.py#L532) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 154 | [pragmeval/squinky-informativeness](src/tasksource/tasks.py#L533) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 155 | [pragmeval/persuasiveness-eloquence](src/tasksource/tasks.py#L534) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 156 | [pragmeval/persuasiveness-relevance](src/tasksource/tasks.py#L535) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 157 | [pragmeval/persuasiveness-specificity](src/tasksource/tasks.py#L536) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 158 | [pragmeval/persuasiveness-strength](src/tasksource/tasks.py#L537) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
-| 159 | [silicone/dyda_da](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 160 | [silicone/dyda_e](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 161 | [silicone/maptask](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 162 | [silicone/oasis](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 163 | [silicone/meld_e](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 164 | [silicone/sem](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 165 | [silicone/meld_s](src/tasksource/tasks.py#L539) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 166 | [silicone/iemocap](src/tasksource/tasks.py#L546) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
-| 167 | [lex_glue/eurlex](src/tasksource/tasks.py#L551) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
-| 168 | [lex_glue/scotus](src/tasksource/tasks.py#L553) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
-| 169 | [lex_glue/ledgar](src/tasksource/tasks.py#L556) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
-| 170 | [lex_glue/unfair_tos](src/tasksource/tasks.py#L558) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) | ✓ |
-| 171 | [lex_glue/case_hold](src/tasksource/tasks.py#L561) | MultipleChoice | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
-| 172 | [language-identification](src/tasksource/tasks.py#L569) | Classification | [papluca/language-identification](https://hf.co/datasets/papluca/language-identification) | ✓ |
-| 173 | [imdb](src/tasksource/tasks.py#L574) | Classification | [stanfordnlp/imdb](https://hf.co/datasets/stanfordnlp/imdb) |  |
-| 174 | [rotten_tomatoes](src/tasksource/tasks.py#L576) | Classification | [cornell-movie-review-data/rotten_tomatoes](https://hf.co/datasets/cornell-movie-review-data/rotten_tomatoes) |  |
-| 175 | [ag_news](src/tasksource/tasks.py#L578) | Classification | [fancyzhx/ag_news](https://hf.co/datasets/fancyzhx/ag_news) |  |
-| 176 | [yelp_review_full/yelp_review_full](src/tasksource/tasks.py#L580) | Classification | [Yelp/yelp_review_full](https://hf.co/datasets/Yelp/yelp_review_full) | ✓ |
-| 177 | [financial_phrasebank/sentences_allagree](src/tasksource/tasks.py#L585) | Classification | [ghbacct/financial-phrasebank-all-agree-classification](https://hf.co/datasets/ghbacct/financial-phrasebank-all-agree-classification) |  |
-| 178 | [poem_sentiment](src/tasksource/tasks.py#L590) | Classification | [google-research-datasets/poem_sentiment](https://hf.co/datasets/google-research-datasets/poem_sentiment) |  |
-| 179 | [emotion](src/tasksource/tasks.py#L592) | Classification | [dair-ai/emotion](https://hf.co/datasets/dair-ai/emotion) |  |
-| 180 | [dbpedia_14/dbpedia_14](src/tasksource/tasks.py#L594) | Classification | [fancyzhx/dbpedia_14](https://hf.co/datasets/fancyzhx/dbpedia_14) |  |
-| 181 | [amazon_polarity/amazon_polarity](src/tasksource/tasks.py#L596) | Classification | [fancyzhx/amazon_polarity](https://hf.co/datasets/fancyzhx/amazon_polarity) |  |
-| 182 | [app_reviews](src/tasksource/tasks.py#L598) | Classification | [sealuzh/app_reviews](https://hf.co/datasets/sealuzh/app_reviews) |  |
-| 183 | [hate_speech18](src/tasksource/tasks.py#L602) | Classification | [tasksource/hate_speech18](https://hf.co/datasets/tasksource/hate_speech18) |  |
-| 184 | [sms_spam](src/tasksource/tasks.py#L608) | Classification | [ucirvine/sms_spam](https://hf.co/datasets/ucirvine/sms_spam) |  |
-| 185 | [humicroedit/subtask-1](src/tasksource/tasks.py#L611) | Classification | [tasksource/humicroedit](https://hf.co/datasets/tasksource/humicroedit) | ✓ |
-| 186 | [humicroedit/subtask-2](src/tasksource/tasks.py#L617) | Classification | [tasksource/humicroedit](https://hf.co/datasets/tasksource/humicroedit) | ✓ |
-| 187 | [snips_built_in_intents](src/tasksource/tasks.py#L622) | Classification | [sonos-nlu-benchmark/snips_built_in_intents](https://hf.co/datasets/sonos-nlu-benchmark/snips_built_in_intents) |  |
-| 188 | [hate_speech_offensive](src/tasksource/tasks.py#L626) | Classification | [tdavidson/hate_speech_offensive](https://hf.co/datasets/tdavidson/hate_speech_offensive) |  |
-| 189 | [yahoo_answers_topics](src/tasksource/tasks.py#L628) | Classification | [community-datasets/yahoo_answers_topics](https://hf.co/datasets/community-datasets/yahoo_answers_topics) |  |
-| 190 | [stackoverflow-questions](src/tasksource/tasks.py#L632) | Classification | [pacovaldez/stackoverflow-questions](https://hf.co/datasets/pacovaldez/stackoverflow-questions) | ✓ |
-| 191 | [hyperpartisan_news](src/tasksource/tasks.py#L638) | Classification | [zapsdcn/hyperpartisan_news](https://hf.co/datasets/zapsdcn/hyperpartisan_news) |  |
-| 192 | [sciie](src/tasksource/tasks.py#L643) | Classification | [zapsdcn/sciie](https://hf.co/datasets/zapsdcn/sciie) |  |
-| 193 | [citation_intent](src/tasksource/tasks.py#L644) | Classification | [zapsdcn/citation_intent](https://hf.co/datasets/zapsdcn/citation_intent) |  |
-| 194 | [go_emotions/simplified](src/tasksource/tasks.py#L646) | Classification | [google-research-datasets/go_emotions](https://hf.co/datasets/google-research-datasets/go_emotions) |  |
-| 195 | [scicite](src/tasksource/tasks.py#L650) | Classification | [tasksource/scicite](https://hf.co/datasets/tasksource/scicite) |  |
-| 196 | [liar](src/tasksource/tasks.py#L652) | Classification | [tasksource/liar](https://hf.co/datasets/tasksource/liar) | ✓ |
-| 197 | [lexical_relation_classification/BLESS](src/tasksource/tasks.py#L663) | Classification | json | ✓ |
-| 198 | [lexical_relation_classification/EVALution](src/tasksource/tasks.py#L663) | Classification | json | ✓ |
-| 199 | [lexical_relation_classification/K&H+N](src/tasksource/tasks.py#L663) | Classification | json | ✓ |
-| 200 | [lexical_relation_classification/ROOT09](src/tasksource/tasks.py#L663) | Classification | json | ✓ |
-| 201 | [lexical_relation_classification/CogALexV](src/tasksource/tasks.py#L689) | Classification | json | ✓ |
-| 202 | [linguisticprobing/subj_number](src/tasksource/tasks.py#L707) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 203 | [linguisticprobing/obj_number](src/tasksource/tasks.py#L708) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 204 | [linguisticprobing/past_present](src/tasksource/tasks.py#L709) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 205 | [linguisticprobing/sentence_length](src/tasksource/tasks.py#L710) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 206 | [linguisticprobing/top_constituents](src/tasksource/tasks.py#L711) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 207 | [linguisticprobing/tree_depth](src/tasksource/tasks.py#L713) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 208 | [linguisticprobing/coordination_inversion](src/tasksource/tasks.py#L714) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 209 | [linguisticprobing/odd_man_out](src/tasksource/tasks.py#L716) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 210 | [linguisticprobing/bigram_shift](src/tasksource/tasks.py#L717) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
-| 211 | [crowdflower/political-media-audience](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 212 | [crowdflower/political-media-message](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 213 | [crowdflower/text_emotion](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 214 | [crowdflower/corporate-messaging](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 215 | [crowdflower/economic-news](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 216 | [crowdflower/airline-sentiment](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 217 | [crowdflower/tweet_global_warming](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 218 | [crowdflower/sentiment_nuclear_power](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 219 | [crowdflower/political-media-bias](src/tasksource/tasks.py#L719) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
-| 220 | [ethics/commonsense](src/tasksource/tasks.py#L744) | Classification | csv |  |
-| 221 | [ethics/deontology](src/tasksource/tasks.py#L752) | Classification | csv |  |
-| 222 | [ethics/justice](src/tasksource/tasks.py#L760) | Classification | csv |  |
-| 223 | [ethics/virtue](src/tasksource/tasks.py#L768) | Classification | [hendrycks/ethics](https://hf.co/datasets/hendrycks/ethics) |  |
-| 224 | [emo/emo2019](src/tasksource/tasks.py#L777) | Classification | [oneonlee/cleansed_emocontext](https://hf.co/datasets/oneonlee/cleansed_emocontext) |  |
-| 225 | [google_wellformed_query](src/tasksource/tasks.py#L783) | Classification | [tasksource/google_wellformed_query](https://hf.co/datasets/tasksource/google_wellformed_query) | ✓ |
-| 226 | [tweets_hate_speech_detection](src/tasksource/tasks.py#L788) | Classification | [tweets-hate-speech-detection/tweets_hate_speech_detection](https://hf.co/datasets/tweets-hate-speech-detection/tweets_hate_speech_detection) |  |
-| 227 | [wnut_17/wnut_17](src/tasksource/tasks.py#L792) | TokenClassification | [flaitenberger/wnut_17](https://hf.co/datasets/flaitenberger/wnut_17) |  |
-| 228 | [ncbi_disease/ncbi_disease](src/tasksource/tasks.py#L795) | TokenClassification | [ncbi/ncbi_disease](https://hf.co/datasets/ncbi/ncbi_disease) |  |
-| 229 | [acronym_identification](src/tasksource/tasks.py#L798) | TokenClassification | [amirveyseh/acronym_identification](https://hf.co/datasets/amirveyseh/acronym_identification) |  |
-| 230 | [jnlpba/jnlpba](src/tasksource/tasks.py#L801) | TokenClassification | [jnlpba/jnlpba](https://hf.co/datasets/jnlpba/jnlpba) |  |
-| 231 | [ontonotes_english/SpeedOfMagic--ontonotes_english](src/tasksource/tasks.py#L808) | TokenClassification | [SpeedOfMagic/ontonotes_english](https://hf.co/datasets/SpeedOfMagic/ontonotes_english) |  |
-| 232 | [blog_authorship_corpus/gender](src/tasksource/tasks.py#L812) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
-| 233 | [blog_authorship_corpus/age](src/tasksource/tasks.py#L814) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
-| 234 | [blog_authorship_corpus/job](src/tasksource/tasks.py#L817) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
-| 235 | [open_question_type](src/tasksource/tasks.py#L828) | Classification | [Korea-MES/open_question_type](https://hf.co/datasets/Korea-MES/open_question_type) |  |
-| 236 | [health_fact](src/tasksource/tasks.py#L830) | Classification | [marcov/health_fact_promptsource](https://hf.co/datasets/marcov/health_fact_promptsource) |  |
-| 237 | [commonsense_qa](src/tasksource/tasks.py#L834) | MultipleChoice | [tau/commonsense_qa](https://hf.co/datasets/tau/commonsense_qa) |  |
-| 238 | [mc_taco](src/tasksource/tasks.py#L840) | Classification | [marcov/mc_taco_promptsource](https://hf.co/datasets/marcov/mc_taco_promptsource) | ✓ |
-| 239 | [ade_corpus_v2/Ade_corpus_v2_classification](src/tasksource/tasks.py#L847) | Classification | [ade-benchmark-corpus/ade_corpus_v2](https://hf.co/datasets/ade-benchmark-corpus/ade_corpus_v2) |  |
-| 240 | [discosense](src/tasksource/tasks.py#L849) | MultipleChoice | json |  |
-| 241 | [circa](src/tasksource/tasks.py#L856) | Classification | [google-research-datasets/circa](https://hf.co/datasets/google-research-datasets/circa) |  |
-| 242 | [code_x_glue_cc_defect_detection](src/tasksource/tasks.py#L861) | Classification | [google/code_x_glue_cc_defect_detection](https://hf.co/datasets/google/code_x_glue_cc_defect_detection) |  |
-| 243 | [phrase_similarity](src/tasksource/tasks.py#L865) | Classification | [Deehan1866/processed_phrase_similarity](https://hf.co/datasets/Deehan1866/processed_phrase_similarity) |  |
-| 244 | [scientific-exaggeration-detection](src/tasksource/tasks.py#L873) | Classification | [copenlu/scientific-exaggeration-detection](https://hf.co/datasets/copenlu/scientific-exaggeration-detection) |  |
-| 245 | [quarel](src/tasksource/tasks.py#L879) | Classification | [community-datasets/quarel](https://hf.co/datasets/community-datasets/quarel) |  |
-| 246 | [fever-evidence-related](src/tasksource/tasks.py#L884) | Classification | [mwong/fever-evidence-related](https://hf.co/datasets/mwong/fever-evidence-related) |  |
-| 247 | [numer_sense](src/tasksource/tasks.py#L887) | Classification | [tasksource/numer_sense](https://hf.co/datasets/tasksource/numer_sense) |  |
-| 248 | [dynasent/dynabench.dynasent.r1.all/r1](src/tasksource/tasks.py#L894) | Classification | [tasksource/dynasent](https://hf.co/datasets/tasksource/dynasent) |  |
-| 249 | [dynasent/dynabench.dynasent.r2.all/r2](src/tasksource/tasks.py#L898) | Classification | [tasksource/dynasent](https://hf.co/datasets/tasksource/dynasent) |  |
-| 250 | [Sarcasm_News_Headline](src/tasksource/tasks.py#L903) | Classification | [raquiba/Sarcasm_News_Headline](https://hf.co/datasets/raquiba/Sarcasm_News_Headline) |  |
-| 251 | [sem_eval_2010_task_8](src/tasksource/tasks.py#L906) | Classification | [SemEvalWorkshop/sem_eval_2010_task_8](https://hf.co/datasets/SemEvalWorkshop/sem_eval_2010_task_8) |  |
-| 252 | [auditor_review](src/tasksource/tasks.py#L908) | Classification | [demo-org/auditor_review](https://hf.co/datasets/demo-org/auditor_review) |  |
-| 253 | [medmcqa](src/tasksource/tasks.py#L912) | MultipleChoice | [openlifescienceai/medmcqa](https://hf.co/datasets/openlifescienceai/medmcqa) |  |
-| 254 | [Dynasent_Disagreement](src/tasksource/tasks.py#L929) | Classification | [RuyuanWan/Dynasent_Disagreement](https://hf.co/datasets/RuyuanWan/Dynasent_Disagreement) | ✓ |
-| 255 | [Politeness_Disagreement](src/tasksource/tasks.py#L931) | Classification | [RuyuanWan/Politeness_Disagreement](https://hf.co/datasets/RuyuanWan/Politeness_Disagreement) | ✓ |
-| 256 | [SBIC_Disagreement](src/tasksource/tasks.py#L933) | Classification | [RuyuanWan/SBIC_Disagreement](https://hf.co/datasets/RuyuanWan/SBIC_Disagreement) | ✓ |
-| 257 | [SChem_Disagreement](src/tasksource/tasks.py#L935) | Classification | [RuyuanWan/SChem_Disagreement](https://hf.co/datasets/RuyuanWan/SChem_Disagreement) | ✓ |
-| 258 | [Dilemmas_Disagreement](src/tasksource/tasks.py#L937) | Classification | [RuyuanWan/Dilemmas_Disagreement](https://hf.co/datasets/RuyuanWan/Dilemmas_Disagreement) | ✓ |
-| 259 | [logiqa](src/tasksource/tasks.py#L940) | MultipleChoice | [fireworks-ai/logiqa](https://hf.co/datasets/fireworks-ai/logiqa) |  |
-| 260 | [wiki_qa](src/tasksource/tasks.py#L949) | Classification | [microsoft/wiki_qa](https://hf.co/datasets/microsoft/wiki_qa) | ✓ |
-| 261 | [cycic_classification](src/tasksource/tasks.py#L951) | Classification | [tasksource/cycic_classification](https://hf.co/datasets/tasksource/cycic_classification) |  |
-| 262 | [cycic_multiplechoice](src/tasksource/tasks.py#L953) | MultipleChoice | [tasksource/cycic_multiplechoice](https://hf.co/datasets/tasksource/cycic_multiplechoice) |  |
-| 263 | [sts-companion](src/tasksource/tasks.py#L957) | Classification | [tasksource/sts-companion](https://hf.co/datasets/tasksource/sts-companion) |  |
-| 264 | [commonsense_qa_2.0](src/tasksource/tasks.py#L960) | Classification | [tasksource/commonsense_qa_2.0](https://hf.co/datasets/tasksource/commonsense_qa_2.0) |  |
-| 265 | [lingnli](src/tasksource/tasks.py#L963) | Classification | [tasksource/lingnli](https://hf.co/datasets/tasksource/lingnli) |  |
-| 266 | [monotonicity-entailment](src/tasksource/tasks.py#L965) | Classification | [tasksource/monotonicity-entailment](https://hf.co/datasets/tasksource/monotonicity-entailment) |  |
-| 267 | [arct](src/tasksource/tasks.py#L968) | MultipleChoice | [tasksource/arct](https://hf.co/datasets/tasksource/arct) |  |
-| 268 | [scinli](src/tasksource/tasks.py#L971) | Classification | [tasksource/scinli](https://hf.co/datasets/tasksource/scinli) |  |
-| 269 | [naturallogic](src/tasksource/tasks.py#L975) | Classification | [tasksource/naturallogic](https://hf.co/datasets/tasksource/naturallogic) |  |
-| 270 | [onestop_qa](src/tasksource/tasks.py#L977) | MultipleChoice | [malmaud/onestop_qa](https://hf.co/datasets/malmaud/onestop_qa) |  |
-| 271 | [moral_stories/full](src/tasksource/tasks.py#L980) | MultipleChoice | [LabHC/moral_stories](https://hf.co/datasets/LabHC/moral_stories) |  |
-| 272 | [prost](src/tasksource/tasks.py#L988) | MultipleChoice | json |  |
-| 273 | [dynahate](src/tasksource/tasks.py#L993) | Classification | [tasksource/dynahate](https://hf.co/datasets/tasksource/dynahate) |  |
-| 274 | [syntactic-augmentation-nli](src/tasksource/tasks.py#L995) | Classification | [tasksource/syntactic-augmentation-nli](https://hf.co/datasets/tasksource/syntactic-augmentation-nli) |  |
-| 275 | [autotnli](src/tasksource/tasks.py#L997) | Classification | [tasksource/autotnli](https://hf.co/datasets/tasksource/autotnli) |  |
-| 276 | [CONDAQA](src/tasksource/tasks.py#L999) | Classification | [lasha-nlp/CONDAQA](https://hf.co/datasets/lasha-nlp/CONDAQA) |  |
-| 277 | [webgpt_comparisons](src/tasksource/tasks.py#L1009) | MultipleChoice | [heegyu/webgpt_comparisons_ko](https://hf.co/datasets/heegyu/webgpt_comparisons_ko) | ✓ |
-| 278 | [synthetic-instruct-gptj-pairwise](src/tasksource/tasks.py#L1017) | MultipleChoice | [Dahoas/synthetic-instruct-gptj-pairwise](https://hf.co/datasets/Dahoas/synthetic-instruct-gptj-pairwise) | ✓ |
-| 279 | [scruples](src/tasksource/tasks.py#L1020) | Classification | [tasksource/scruples](https://hf.co/datasets/tasksource/scruples) | ✓ |
-| 280 | [wouldyourather](src/tasksource/tasks.py#L1022) | MultipleChoice | [tasksource/wouldyourather](https://hf.co/datasets/tasksource/wouldyourather) | ✓ |
-| 281 | [defeasible-nli/snli](src/tasksource/tasks.py#L1030) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
-| 282 | [defeasible-nli/atomic](src/tasksource/tasks.py#L1030) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
-| 283 | [defeasible-nli/social](src/tasksource/tasks.py#L1033) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
-| 284 | [help-nli](src/tasksource/tasks.py#L1036) | Classification | [tasksource/help-nli](https://hf.co/datasets/tasksource/help-nli) |  |
-| 285 | [nli-veridicality-transitivity](src/tasksource/tasks.py#L1039) | Classification | [tasksource/nli-veridicality-transitivity](https://hf.co/datasets/tasksource/nli-veridicality-transitivity) |  |
-| 286 | [lonli](src/tasksource/tasks.py#L1042) | Classification | [tasksource/lonli](https://hf.co/datasets/tasksource/lonli) |  |
-| 287 | [dadc-limit-nli](src/tasksource/tasks.py#L1045) | Classification | [tasksource/dadc-limit-nli](https://hf.co/datasets/tasksource/dadc-limit-nli) |  |
-| 288 | [FLUTE](src/tasksource/tasks.py#L1048) | Classification | [ColumbiaNLP/FLUTE](https://hf.co/datasets/ColumbiaNLP/FLUTE) |  |
-| 289 | [strategy-qa](src/tasksource/tasks.py#L1051) | Classification | [tasksource/strategy-qa](https://hf.co/datasets/tasksource/strategy-qa) |  |
-| 290 | [summarize_from_feedback/comparisons](src/tasksource/tasks.py#L1054) | MultipleChoice | [vwxyzjn/summarize_from_feedback_oai_preprocessing](https://hf.co/datasets/vwxyzjn/summarize_from_feedback_oai_preprocessing) | ✓ |
-| 291 | [folio](src/tasksource/tasks.py#L1062) | Classification | [tasksource/folio](https://hf.co/datasets/tasksource/folio) |  |
-| 292 | [tomi-nli](src/tasksource/tasks.py#L1066) | Classification | [tasksource/tomi-nli](https://hf.co/datasets/tasksource/tomi-nli) |  |
-| 293 | [avicenna](src/tasksource/tasks.py#L1069) | Classification | [tasksource/avicenna](https://hf.co/datasets/tasksource/avicenna) | ✓ |
-| 294 | [SHP](src/tasksource/tasks.py#L1072) | MultipleChoice | [stanfordnlp/SHP](https://hf.co/datasets/stanfordnlp/SHP) | ✓ |
-| 295 | [MedQA-USMLE-4-options-hf](src/tasksource/tasks.py#L1080) | MultipleChoice | [GBaker/MedQA-USMLE-4-options-hf](https://hf.co/datasets/GBaker/MedQA-USMLE-4-options-hf) |  |
-| 296 | [wikimedqa/medwiki](src/tasksource/tasks.py#L1083) | MultipleChoice | [sileod/wikimedqa](https://hf.co/datasets/sileod/wikimedqa) |  |
-| 297 | [cicero](src/tasksource/tasks.py#L1092) | MultipleChoice | [declare-lab/cicero](https://hf.co/datasets/declare-lab/cicero) |  |
-| 298 | [CREAK](src/tasksource/tasks.py#L1096) | Classification | [amydeng2000/CREAK](https://hf.co/datasets/amydeng2000/CREAK) |  |
-| 299 | [mutual](src/tasksource/tasks.py#L1099) | MultipleChoice | [tasksource/mutual](https://hf.co/datasets/tasksource/mutual) |  |
-| 300 | [puzzte](src/tasksource/tasks.py#L1103) | Classification | [tasksource/puzzte](https://hf.co/datasets/tasksource/puzzte) |  |
-| 301 | [implicatures](src/tasksource/tasks.py#L1108) | MultipleChoice | [tasksource/implicatures](https://hf.co/datasets/tasksource/implicatures) |  |
-| 302 | [race/high](src/tasksource/tasks.py#L1113) | MultipleChoice | [ehovy/race](https://hf.co/datasets/ehovy/race) |  |
-| 303 | [race/middle](src/tasksource/tasks.py#L1113) | MultipleChoice | [ehovy/race](https://hf.co/datasets/ehovy/race) |  |
-| 304 | [race-c](src/tasksource/tasks.py#L1117) | MultipleChoice | [tasksource/race-c](https://hf.co/datasets/tasksource/race-c) |  |
-| 305 | [spartqa-yn](src/tasksource/tasks.py#L1120) | Classification | [tasksource/spartqa-yn](https://hf.co/datasets/tasksource/spartqa-yn) |  |
-| 306 | [spartqa-mchoice](src/tasksource/tasks.py#L1123) | MultipleChoice | [tasksource/spartqa-mchoice](https://hf.co/datasets/tasksource/spartqa-mchoice) |  |
-| 307 | [temporal-nli](src/tasksource/tasks.py#L1126) | Classification | [tasksource/temporal-nli](https://hf.co/datasets/tasksource/temporal-nli) |  |
-| 308 | [riddle_sense](src/tasksource/tasks.py#L1129) | MultipleChoice | [jeggers/riddle_sense](https://hf.co/datasets/jeggers/riddle_sense) |  |
-| 309 | [clcd-english](src/tasksource/tasks.py#L1134) | Classification | [tasksource/clcd-english](https://hf.co/datasets/tasksource/clcd-english) |  |
-| 310 | [twentyquestions](src/tasksource/tasks.py#L1146) | Classification | [tasksource/twentyquestions](https://hf.co/datasets/tasksource/twentyquestions) |  |
-| 311 | [reclor](src/tasksource/tasks.py#L1151) | MultipleChoice | [tasksource/reclor](https://hf.co/datasets/tasksource/reclor) |  |
-| 312 | [counterfactually-augmented-imdb](src/tasksource/tasks.py#L1154) | Classification | [tasksource/counterfactually-augmented-imdb](https://hf.co/datasets/tasksource/counterfactually-augmented-imdb) |  |
-| 313 | [counterfactually-augmented-snli](src/tasksource/tasks.py#L1157) | Classification | [tasksource/counterfactually-augmented-snli](https://hf.co/datasets/tasksource/counterfactually-augmented-snli) |  |
-| 314 | [cnli](src/tasksource/tasks.py#L1160) | Classification | [tasksource/cnli](https://hf.co/datasets/tasksource/cnli) |  |
-| 315 | [boolq-natural-perturbations](src/tasksource/tasks.py#L1163) | Classification | [tasksource/boolq-natural-perturbations](https://hf.co/datasets/tasksource/boolq-natural-perturbations) |  |
-| 316 | [acceptability-prediction](src/tasksource/tasks.py#L1167) | Classification | [tasksource/acceptability-prediction](https://hf.co/datasets/tasksource/acceptability-prediction) | ✓ |
-| 317 | [equate](src/tasksource/tasks.py#L1171) | Classification | [tasksource/equate](https://hf.co/datasets/tasksource/equate) |  |
-| 318 | [ScienceQA_text_only](src/tasksource/tasks.py#L1174) | MultipleChoice | [tasksource/ScienceQA_text_only](https://hf.co/datasets/tasksource/ScienceQA_text_only) |  |
-| 319 | [ekar_english](src/tasksource/tasks.py#L1177) | MultipleChoice | [Jiangjie/ekar_english](https://hf.co/datasets/Jiangjie/ekar_english) | ✓ |
-| 320 | [implicit-hate-stg1](src/tasksource/tasks.py#L1181) | Classification | [tasksource/implicit-hate-stg1](https://hf.co/datasets/tasksource/implicit-hate-stg1) |  |
-| 321 | [chaos-mnli-ambiguity](src/tasksource/tasks.py#L1184) | Classification | [tasksource/chaos-mnli-ambiguity](https://hf.co/datasets/tasksource/chaos-mnli-ambiguity) | ✓ |
-| 322 | [headline_cause/en_simple](src/tasksource/tasks.py#L1188) | Classification | json |  |
-| 323 | [logiqa-2.0-nli](src/tasksource/tasks.py#L1193) | Classification | [tasksource/logiqa-2.0-nli](https://hf.co/datasets/tasksource/logiqa-2.0-nli) |  |
-| 324 | [oasst2_dense_flat/quality](src/tasksource/tasks.py#L1198) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
-| 325 | [oasst2_dense_flat/toxicity](src/tasksource/tasks.py#L1200) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
-| 326 | [oasst2_dense_flat/helpfulness](src/tasksource/tasks.py#L1202) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
-| 327 | [mindgames](src/tasksource/tasks.py#L1205) | Classification | [sileod/mindgames](https://hf.co/datasets/sileod/mindgames) |  |
-| 328 | [universal_dependencies/en_partut/deprel](src/tasksource/tasks.py#L1219) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
-| 329 | [universal_dependencies/en_lines/deprel](src/tasksource/tasks.py#L1219) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
-| 330 | [universal_dependencies/en_gum/deprel](src/tasksource/tasks.py#L1219) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
-| 331 | [universal_dependencies/en_ewt/deprel](src/tasksource/tasks.py#L1219) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
-| 332 | [ambient](src/tasksource/tasks.py#L1225) | Classification | [tasksource/ambient](https://hf.co/datasets/tasksource/ambient) | ✓ |
-| 333 | [path-naturalness-prediction](src/tasksource/tasks.py#L1228) | MultipleChoice | [tasksource/path-naturalness-prediction](https://hf.co/datasets/tasksource/path-naturalness-prediction) | ✓ |
-| 334 | [civil_comments/toxicity](src/tasksource/tasks.py#L1238) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 335 | [civil_comments/severe_toxicity](src/tasksource/tasks.py#L1239) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 336 | [civil_comments/obscene](src/tasksource/tasks.py#L1240) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 337 | [civil_comments/threat](src/tasksource/tasks.py#L1241) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 338 | [civil_comments/insult](src/tasksource/tasks.py#L1242) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 339 | [civil_comments/identity_attack](src/tasksource/tasks.py#L1243) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 340 | [civil_comments/sexual_explicit](src/tasksource/tasks.py#L1244) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
-| 341 | [cloth](src/tasksource/tasks.py#L1246) | MultipleChoice | [AndyChiang/cloth](https://hf.co/datasets/AndyChiang/cloth) |  |
-| 342 | [dgen](src/tasksource/tasks.py#L1247) | MultipleChoice | [AndyChiang/dgen](https://hf.co/datasets/AndyChiang/dgen) |  |
-| 343 | [I2D2](src/tasksource/tasks.py#L1249) | Classification | [tasksource/I2D2](https://hf.co/datasets/tasksource/I2D2) |  |
-| 344 | [args_me](src/tasksource/tasks.py#L1251) | Classification | [webis/args_me](https://hf.co/datasets/webis/args_me) |  |
-| 345 | [Touche23-ValueEval](src/tasksource/tasks.py#L1254) | Classification | csv |  |
-| 346 | [starcon](src/tasksource/tasks.py#L1262) | Classification | [tasksource/starcon](https://hf.co/datasets/tasksource/starcon) |  |
-| 347 | [banking77](src/tasksource/tasks.py#L1264) | Classification | [legacy-datasets/banking77](https://hf.co/datasets/legacy-datasets/banking77) |  |
-| 348 | [it-support-tickets](src/tasksource/tasks.py#L1266) | Classification | [tasksource/it-support-tickets](https://hf.co/datasets/tasksource/it-support-tickets) |  |
-| 349 | [ConTRoL-nli](src/tasksource/tasks.py#L1270) | Classification | [tasksource/ConTRoL-nli](https://hf.co/datasets/tasksource/ConTRoL-nli) |  |
-| 350 | [tracie](src/tasksource/tasks.py#L1271) | Classification | [tasksource/tracie](https://hf.co/datasets/tasksource/tracie) |  |
-| 351 | [sherliic](src/tasksource/tasks.py#L1272) | Classification | [tasksource/sherliic](https://hf.co/datasets/tasksource/sherliic) |  |
-| 352 | [sen-making/1](src/tasksource/tasks.py#L1274) | MultipleChoice | [tasksource/sen-making](https://hf.co/datasets/tasksource/sen-making) | ✓ |
-| 353 | [sen-making/2](src/tasksource/tasks.py#L1278) | MultipleChoice | [tasksource/sen-making](https://hf.co/datasets/tasksource/sen-making) | ✓ |
-| 354 | [winowhy](src/tasksource/tasks.py#L1281) | Classification | [tasksource/winowhy](https://hf.co/datasets/tasksource/winowhy) | ✓ |
-| 355 | [robustLR](src/tasksource/tasks.py#L1285) | Classification | [tasksource/robustLR](https://hf.co/datasets/tasksource/robustLR) |  |
-| 356 | [clutrr](src/tasksource/tasks.py#L1287) | Classification | [tasksource/clutrr](https://hf.co/datasets/tasksource/clutrr) |  |
-| 357 | [logical-fallacy](src/tasksource/tasks.py#L1289) | Classification | [tasksource/logical-fallacy](https://hf.co/datasets/tasksource/logical-fallacy) |  |
-| 358 | [parade](src/tasksource/tasks.py#L1291) | Classification | [tasksource/parade](https://hf.co/datasets/tasksource/parade) |  |
-| 359 | [cladder](src/tasksource/tasks.py#L1293) | Classification | [tasksource/cladder](https://hf.co/datasets/tasksource/cladder) |  |
-| 360 | [subjectivity](src/tasksource/tasks.py#L1295) | Classification | [tasksource/subjectivity](https://hf.co/datasets/tasksource/subjectivity) |  |
-| 361 | [MOH](src/tasksource/tasks.py#L1297) | Classification | [tasksource/MOH](https://hf.co/datasets/tasksource/MOH) |  |
-| 362 | [VUAC](src/tasksource/tasks.py#L1298) | Classification | [tasksource/VUAC](https://hf.co/datasets/tasksource/VUAC) |  |
-| 363 | [TroFi](src/tasksource/tasks.py#L1299) | Classification | parquet |  |
-| 364 | [sharc](src/tasksource/tasks.py#L1306) | Classification | [tasksource/sharc](https://hf.co/datasets/tasksource/sharc) |  |
-| 365 | [conceptrules_v2](src/tasksource/tasks.py#L1310) | Classification | [tasksource/conceptrules_v2](https://hf.co/datasets/tasksource/conceptrules_v2) | ✓ |
-| 366 | [disrpt/eng.dep.scidtb.rels](src/tasksource/tasks.py#L1312) | Classification | [multilingual-discourse-hub/disrpt](https://hf.co/datasets/multilingual-discourse-hub/disrpt) |  |
-| 367 | [conll2000](src/tasksource/tasks.py#L1314) | TokenClassification | [eriktks/conll2000](https://hf.co/datasets/eriktks/conll2000) |  |
-| 368 | [few-nerd/supervised](src/tasksource/tasks.py#L1317) | TokenClassification | [DFKI-SLT/few-nerd](https://hf.co/datasets/DFKI-SLT/few-nerd) |  |
-| 369 | [finer-139](src/tasksource/tasks.py#L1318) | TokenClassification | [nlpaueb/finer-139](https://hf.co/datasets/nlpaueb/finer-139) |  |
-| 370 | [zero-shot-label-nli](src/tasksource/tasks.py#L1321) | Classification | [tasksource/zero-shot-label-nli](https://hf.co/datasets/tasksource/zero-shot-label-nli) |  |
-| 371 | [com2sense](src/tasksource/tasks.py#L1323) | Classification | [tasksource/com2sense](https://hf.co/datasets/tasksource/com2sense) |  |
-| 372 | [scone](src/tasksource/tasks.py#L1325) | Classification | [tasksource/scone](https://hf.co/datasets/tasksource/scone) |  |
-| 373 | [winodict](src/tasksource/tasks.py#L1327) | MultipleChoice | [tasksource/winodict](https://hf.co/datasets/tasksource/winodict) |  |
-| 374 | [fool-me-twice](src/tasksource/tasks.py#L1329) | Classification | [tasksource/fool-me-twice](https://hf.co/datasets/tasksource/fool-me-twice) |  |
-| 375 | [monli](src/tasksource/tasks.py#L1333) | Classification | [tasksource/monli](https://hf.co/datasets/tasksource/monli) |  |
-| 376 | [corr2cause](src/tasksource/tasks.py#L1335) | Classification | [tasksource/corr2cause](https://hf.co/datasets/tasksource/corr2cause) |  |
-| 377 | [lsat_qa/all](src/tasksource/tasks.py#L1337) | MultipleChoice | [lighteval/lsat_qa](https://hf.co/datasets/lighteval/lsat_qa) |  |
-| 378 | [apt](src/tasksource/tasks.py#L1339) | Classification | [tasksource/apt](https://hf.co/datasets/tasksource/apt) |  |
-| 379 | [twitter-financial-news-sentiment](src/tasksource/tasks.py#L1342) | Classification | [zeroshot/twitter-financial-news-sentiment](https://hf.co/datasets/zeroshot/twitter-financial-news-sentiment) |  |
-| 380 | [icl-symbol-tuning-instruct](src/tasksource/tasks.py#L1349) | Classification | [tasksource/icl-symbol-tuning-instruct](https://hf.co/datasets/tasksource/icl-symbol-tuning-instruct) | ✓ |
-| 381 | [SpaceNLI](src/tasksource/tasks.py#L1355) | Classification | [tasksource/SpaceNLI](https://hf.co/datasets/tasksource/SpaceNLI) |  |
-| 382 | [propsegment/nli](src/tasksource/tasks.py#L1357) | Classification | json |  |
-| 383 | [HatemojiBuild](src/tasksource/tasks.py#L1366) | Classification | [HannahRoseKirk/HatemojiBuild](https://hf.co/datasets/HannahRoseKirk/HatemojiBuild) |  |
-| 384 | [regset](src/tasksource/tasks.py#L1369) | Classification | [tasksource/regset](https://hf.co/datasets/tasksource/regset) | ✓ |
-| 385 | [esci](src/tasksource/tasks.py#L1375) | Classification | [tasksource/esci](https://hf.co/datasets/tasksource/esci) |  |
-| 386 | [chatbot_arena_conversations](src/tasksource/tasks.py#L1394) | MultipleChoice | [lmsys/chatbot_arena_conversations](https://hf.co/datasets/lmsys/chatbot_arena_conversations) | ✓ |
-| 387 | [dnd_style_intents](src/tasksource/tasks.py#L1400) | Classification | [neurae/dnd_style_intents](https://hf.co/datasets/neurae/dnd_style_intents) |  |
-| 388 | [FLD.v2/default](src/tasksource/tasks.py#L1403) | Classification | [hitachi-nlp/FLD.v2](https://hf.co/datasets/hitachi-nlp/FLD.v2) |  |
-| 389 | [FLD.v2/star](src/tasksource/tasks.py#L1406) | Classification | [hitachi-nlp/FLD.v2](https://hf.co/datasets/hitachi-nlp/FLD.v2) |  |
-| 390 | [SDOH-NLI](src/tasksource/tasks.py#L1409) | Classification | [tasksource/SDOH-NLI](https://hf.co/datasets/tasksource/SDOH-NLI) |  |
-| 391 | [scifact_entailment](src/tasksource/tasks.py#L1412) | Classification | [tasksource/scifact_entailment](https://hf.co/datasets/tasksource/scifact_entailment) |  |
-| 392 | [feasibilityQA](src/tasksource/tasks.py#L1416) | Classification | [tasksource/feasibilityQA](https://hf.co/datasets/tasksource/feasibilityQA) |  |
-| 393 | [simple_pair](src/tasksource/tasks.py#L1419) | Classification | [tasksource/simple_pair](https://hf.co/datasets/tasksource/simple_pair) |  |
-| 394 | [AdjectiveScaleProbe-nli](src/tasksource/tasks.py#L1420) | Classification | [tasksource/AdjectiveScaleProbe-nli](https://hf.co/datasets/tasksource/AdjectiveScaleProbe-nli) |  |
-| 395 | [resnli](src/tasksource/tasks.py#L1421) | Classification | [tasksource/resnli](https://hf.co/datasets/tasksource/resnli) |  |
-| 396 | [SpaRTUN](src/tasksource/tasks.py#L1423) | MultipleChoice | [tasksource/SpaRTUN](https://hf.co/datasets/tasksource/SpaRTUN) |  |
-| 397 | [ReSQ](src/tasksource/tasks.py#L1428) | MultipleChoice | [tasksource/ReSQ](https://hf.co/datasets/tasksource/ReSQ) |  |
-| 398 | [semantic_fragments_nli](src/tasksource/tasks.py#L1433) | Classification | [tasksource/semantic_fragments_nli](https://hf.co/datasets/tasksource/semantic_fragments_nli) |  |
-| 399 | [dataset_train_nli](src/tasksource/tasks.py#L1436) | Classification | [MoritzLaurer/dataset_train_nli](https://hf.co/datasets/MoritzLaurer/dataset_train_nli) |  |
-| 400 | [stepgame](src/tasksource/tasks.py#L1441) | Classification | [tasksource/stepgame](https://hf.co/datasets/tasksource/stepgame) |  |
-| 401 | [nlgraph](src/tasksource/tasks.py#L1449) | Classification | [tasksource/nlgraph](https://hf.co/datasets/tasksource/nlgraph) |  |
-| 402 | [oasst2_pairwise_rlhf_reward](src/tasksource/tasks.py#L1453) | MultipleChoice | [tasksource/oasst2_pairwise_rlhf_reward](https://hf.co/datasets/tasksource/oasst2_pairwise_rlhf_reward) | ✓ |
-| 403 | [hh-rlhf/helpful-rejection-sampled](src/tasksource/tasks.py#L1464) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
-| 404 | [hh-rlhf/helpful-online](src/tasksource/tasks.py#L1464) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
-| 405 | [hh-rlhf/helpful-base](src/tasksource/tasks.py#L1464) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
-| 406 | [hh-rlhf/harmless-base](src/tasksource/tasks.py#L1468) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
-| 407 | [ruletaker](src/tasksource/tasks.py#L1472) | Classification | [tasksource/ruletaker](https://hf.co/datasets/tasksource/ruletaker) | ✓ |
-| 408 | [PARARULE-Plus](src/tasksource/tasks.py#L1476) | Classification | [qbao775/PARARULE-Plus](https://hf.co/datasets/qbao775/PARARULE-Plus) | ✓ |
-| 409 | [proofwriter](src/tasksource/tasks.py#L1480) | Classification | [tasksource/proofwriter](https://hf.co/datasets/tasksource/proofwriter) |  |
-| 410 | [logical-entailment](src/tasksource/tasks.py#L1483) | Classification | [tasksource/logical-entailment](https://hf.co/datasets/tasksource/logical-entailment) |  |
-| 411 | [nope](src/tasksource/tasks.py#L1485) | Classification | [tasksource/nope](https://hf.co/datasets/tasksource/nope) |  |
-| 412 | [LogicNLI](src/tasksource/tasks.py#L1489) | Classification | [tasksource/LogicNLI](https://hf.co/datasets/tasksource/LogicNLI) |  |
-| 413 | [contract-nli/contractnli_a/seg](src/tasksource/tasks.py#L1491) | Classification | [tasksource/contract-nli](https://hf.co/datasets/tasksource/contract-nli) |  |
-| 414 | [contract-nli/contractnli_b/full](src/tasksource/tasks.py#L1493) | Classification | [tasksource/contract-nli](https://hf.co/datasets/tasksource/contract-nli) |  |
-| 415 | [nli4ct_semeval2024](src/tasksource/tasks.py#L1495) | Classification | [AshtonIsNotHere/nli4ct_semeval2024](https://hf.co/datasets/AshtonIsNotHere/nli4ct_semeval2024) |  |
-| 416 | [lsat-ar](src/tasksource/tasks.py#L1498) | MultipleChoice | [tasksource/lsat-ar](https://hf.co/datasets/tasksource/lsat-ar) |  |
-| 417 | [lsat-rc](src/tasksource/tasks.py#L1503) | MultipleChoice | [tasksource/lsat-rc](https://hf.co/datasets/tasksource/lsat-rc) |  |
-| 418 | [biosift-nli](src/tasksource/tasks.py#L1508) | Classification | [AshtonIsNotHere/biosift-nli](https://hf.co/datasets/AshtonIsNotHere/biosift-nli) |  |
-| 419 | [brainteasers/WP](src/tasksource/tasks.py#L1512) | MultipleChoice | [tasksource/brainteasers](https://hf.co/datasets/tasksource/brainteasers) |  |
-| 420 | [brainteasers/SP](src/tasksource/tasks.py#L1512) | MultipleChoice | [tasksource/brainteasers](https://hf.co/datasets/tasksource/brainteasers) |  |
-| 421 | [toxigen-data/annotated](src/tasksource/tasks.py#L1518) | Classification | [skg/toxigen-data](https://hf.co/datasets/skg/toxigen-data) |  |
-| 422 | [persuasion](src/tasksource/tasks.py#L1529) | Classification | [Anthropic/persuasion](https://hf.co/datasets/Anthropic/persuasion) |  |
-| 423 | [AmbigNQ-clarifying-question](src/tasksource/tasks.py#L1535) | Classification | [erbacher/AmbigNQ-clarifying-question](https://hf.co/datasets/erbacher/AmbigNQ-clarifying-question) |  |
-| 424 | [SIGA-nli](src/tasksource/tasks.py#L1538) | Classification | [tasksource/SIGA-nli](https://hf.co/datasets/tasksource/SIGA-nli) |  |
-| 425 | [FOL-nli](src/tasksource/tasks.py#L1540) | Classification | [unigram/FOL-nli](https://hf.co/datasets/unigram/FOL-nli) |  |
-| 426 | [goal-step-wikihow/goal](src/tasksource/tasks.py#L1542) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
-| 427 | [goal-step-wikihow/step](src/tasksource/tasks.py#L1545) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
-| 428 | [goal-step-wikihow/order](src/tasksource/tasks.py#L1548) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
-| 429 | [PARADISE](src/tasksource/tasks.py#L1551) | MultipleChoice | [GGLab/PARADISE](https://hf.co/datasets/GGLab/PARADISE) |  |
-| 430 | [doc-nli](src/tasksource/tasks.py#L1554) | Classification | [tasksource/doc-nli](https://hf.co/datasets/tasksource/doc-nli) |  |
-| 431 | [mctest-nli](src/tasksource/tasks.py#L1556) | Classification | [tasksource/mctest-nli](https://hf.co/datasets/tasksource/mctest-nli) |  |
-| 432 | [patent-phrase-similarity](src/tasksource/tasks.py#L1558) | Classification | [tasksource/patent-phrase-similarity](https://hf.co/datasets/tasksource/patent-phrase-similarity) |  |
-| 433 | [natural-language-satisfiability](src/tasksource/tasks.py#L1560) | Classification | [tasksource/natural-language-satisfiability](https://hf.co/datasets/tasksource/natural-language-satisfiability) |  |
-| 434 | [idioms-nli](src/tasksource/tasks.py#L1562) | Classification | [tasksource/idioms-nli](https://hf.co/datasets/tasksource/idioms-nli) |  |
-| 435 | [lifecycle-entailment](src/tasksource/tasks.py#L1564) | Classification | [tasksource/lifecycle-entailment](https://hf.co/datasets/tasksource/lifecycle-entailment) |  |
-| 436 | [safe-guard-prompt-injection](src/tasksource/tasks.py#L1571) | Classification | [xTRam1/safe-guard-prompt-injection](https://hf.co/datasets/xTRam1/safe-guard-prompt-injection) | ✓ |
-| 437 | [prompt-injections](src/tasksource/tasks.py#L1577) | Classification | [deepset/prompt-injections](https://hf.co/datasets/deepset/prompt-injections) | ✓ |
-| 438 | [prompt-injection-dataset](src/tasksource/tasks.py#L1583) | Classification | [S-Labs/prompt-injection-dataset](https://hf.co/datasets/S-Labs/prompt-injection-dataset) | ✓ |
-| 439 | [Prompt-injection-dataset/full](src/tasksource/tasks.py#L1589) | Classification | [neuralchemy/Prompt-injection-dataset](https://hf.co/datasets/neuralchemy/Prompt-injection-dataset) | ✓ |
-| 440 | [PromptShield](src/tasksource/tasks.py#L1595) | Classification | [hendzh/PromptShield](https://hf.co/datasets/hendzh/PromptShield) | ✓ |
-| 441 | [shell-safety-v2](src/tasksource/tasks.py#L1601) | Classification | [tomngdev/shell-safety-v2](https://hf.co/datasets/tomngdev/shell-safety-v2) | ✓ |
-| 442 | [agent_action_safety](src/tasksource/tasks.py#L1606) | Classification | json | ✓ |
-| 443 | [ShellRisk-Bench](src/tasksource/tasks.py#L1621) | Classification | [kontext-security/ShellRisk-Bench](https://hf.co/datasets/kontext-security/ShellRisk-Bench) | ✓ |
-| 444 | [wildguardmix-cleaned/prompt_harm](src/tasksource/tasks.py#L1626) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
-| 445 | [wildguardmix-cleaned/response_harm](src/tasksource/tasks.py#L1631) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
-| 446 | [wildguardmix-cleaned/response_refusal](src/tasksource/tasks.py#L1636) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
-| 447 | [BeaverTails](src/tasksource/tasks.py#L1641) | Classification | [PKU-Alignment/BeaverTails](https://hf.co/datasets/PKU-Alignment/BeaverTails) | ✓ |
-| 448 | [toxic-chat/toxicchat0124/toxicity](src/tasksource/tasks.py#L1648) | Classification | [lmsys/toxic-chat](https://hf.co/datasets/lmsys/toxic-chat) | ✓ |
-| 449 | [toxic-chat/toxicchat0124/jailbreaking](src/tasksource/tasks.py#L1653) | Classification | [lmsys/toxic-chat](https://hf.co/datasets/lmsys/toxic-chat) | ✓ |
-| 450 | [clinc_oos/plus](src/tasksource/tasks.py#L1659) | Classification | [clinc/clinc_oos](https://hf.co/datasets/clinc/clinc_oos) |  |
-| 451 | [IntentGrasp/all](src/tasksource/tasks.py#L1675) | MultipleChoice | [yuweiyin/IntentGrasp](https://hf.co/datasets/yuweiyin/IntentGrasp) |  |
-| 452 | [few_rel/default](src/tasksource/tasks.py#L1713) | Classification | [tasksource/few_rel](https://hf.co/datasets/tasksource/few_rel) |  |
-| 453 | [docred](src/tasksource/tasks.py#L1759) | Classification | json |  |
-| 454 | [chemprot/chemprot_full_source](src/tasksource/tasks.py#L1787) | Classification | [bigbio/chemprot](https://hf.co/datasets/bigbio/chemprot) |  |
-| 455 | [PKU-SafeRLHF/helpfulness](src/tasksource/tasks.py#L1792) | MultipleChoice | [PKU-Alignment/PKU-SafeRLHF](https://hf.co/datasets/PKU-Alignment/PKU-SafeRLHF) | ✓ |
-| 456 | [PKU-SafeRLHF/safety](src/tasksource/tasks.py#L1797) | MultipleChoice | [PKU-Alignment/PKU-SafeRLHF](https://hf.co/datasets/PKU-Alignment/PKU-SafeRLHF) | ✓ |
-| 457 | [HelpSteer/helpfulness](src/tasksource/tasks.py#L1819) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
-| 458 | [HelpSteer/correctness](src/tasksource/tasks.py#L1820) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
-| 459 | [HelpSteer/coherence](src/tasksource/tasks.py#L1821) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
-| 460 | [HelpSteer/complexity](src/tasksource/tasks.py#L1822) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
-| 461 | [HelpSteer/verbosity](src/tasksource/tasks.py#L1823) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
-| 462 | [HelpSteer2/helpfulness](src/tasksource/tasks.py#L1825) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
-| 463 | [HelpSteer2/correctness](src/tasksource/tasks.py#L1826) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
-| 464 | [HelpSteer2/coherence](src/tasksource/tasks.py#L1827) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
-| 465 | [HelpSteer2/complexity](src/tasksource/tasks.py#L1828) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
-| 466 | [HelpSteer2/verbosity](src/tasksource/tasks.py#L1829) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
-| 467 | [HelpSteer3/preference](src/tasksource/tasks.py#L1834) | MultipleChoice | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
-| 468 | [HelpSteer3/principle](src/tasksource/tasks.py#L1839) | Classification | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) |  |
-| 469 | [HelpSteer3/edit_quality](src/tasksource/tasks.py#L1844) | MultipleChoice | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
-| 470 | [HelpSteer3/feedback](src/tasksource/tasks.py#L1867) | Classification | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
-| 471 | [MSciNLI](src/tasksource/tasks.py#L1872) | Classification | [sadat2307/MSciNLI](https://hf.co/datasets/sadat2307/MSciNLI) |  |
-| 472 | [UltraFeedback-paired](src/tasksource/tasks.py#L1875) | MultipleChoice | [pushpdeep/UltraFeedback-paired](https://hf.co/datasets/pushpdeep/UltraFeedback-paired) | ✓ |
-| 473 | [prm800k_dpo/solution](src/tasksource/tasks.py#L1879) | MultipleChoice | [tasksource/prm800k_dpo](https://hf.co/datasets/tasksource/prm800k_dpo) | ✓ |
-| 474 | [prm800k_dpo/step](src/tasksource/tasks.py#L1882) | MultipleChoice | [tasksource/prm800k_dpo](https://hf.co/datasets/tasksource/prm800k_dpo) | ✓ |
-| 475 | [AES2-essay-scoring](src/tasksource/tasks.py#L1886) | Classification | [tasksource/AES2-essay-scoring](https://hf.co/datasets/tasksource/AES2-essay-scoring) | ✓ |
-| 476 | [argument-feedback](src/tasksource/tasks.py#L1890) | Classification | [tasksource/argument-feedback](https://hf.co/datasets/tasksource/argument-feedback) | ✓ |
-| 477 | [english-grading/cohesion](src/tasksource/tasks.py#L1897) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 478 | [english-grading/syntax](src/tasksource/tasks.py#L1898) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 479 | [english-grading/vocabulary](src/tasksource/tasks.py#L1899) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 480 | [english-grading/phraseology](src/tasksource/tasks.py#L1900) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 481 | [english-grading/grammar](src/tasksource/tasks.py#L1901) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 482 | [english-grading/conventions](src/tasksource/tasks.py#L1902) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
-| 483 | [wice](src/tasksource/tasks.py#L1904) | Classification | [tasksource/wice](https://hf.co/datasets/tasksource/wice) |  |
-| 484 | [hover](src/tasksource/tasks.py#L1907) | Classification | [Dzeniks/hover](https://hf.co/datasets/Dzeniks/hover) |  |
-| 485 | [hover-3way/nli](src/tasksource/tasks.py#L1911) | Classification | [Dzeniks/hover-3way](https://hf.co/datasets/Dzeniks/hover-3way) |  |
-| 486 | [tasksource_dpo_pairs](src/tasksource/tasks.py#L1914) | MultipleChoice | [tasksource/tasksource_dpo_pairs](https://hf.co/datasets/tasksource/tasksource_dpo_pairs) | ✓ |
-| 487 | [seahorse_summarization_evaluation](src/tasksource/tasks.py#L1917) | Classification | [tasksource/seahorse_summarization_evaluation](https://hf.co/datasets/tasksource/seahorse_summarization_evaluation) |  |
-| 488 | [missing-item-prediction/contrastive](src/tasksource/tasks.py#L1920) | Classification | [sileod/missing-item-prediction](https://hf.co/datasets/sileod/missing-item-prediction) |  |
-| 489 | [jigsaw_toxicity](src/tasksource/tasks.py#L1924) | Classification | [tasksource/jigsaw_toxicity](https://hf.co/datasets/tasksource/jigsaw_toxicity) |  |
-| 490 | [Pol_NLI](src/tasksource/tasks.py#L1927) | Classification | [mlburnham/Pol_NLI](https://hf.co/datasets/mlburnham/Pol_NLI) |  |
-| 491 | [synthetic-retrieval-NLI/binary](src/tasksource/tasks.py#L1930) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
-| 492 | [synthetic-retrieval-NLI/position](src/tasksource/tasks.py#L1930) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
-| 493 | [synthetic-retrieval-NLI/count](src/tasksource/tasks.py#L1930) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
-| 494 | [github-issue-similarity](src/tasksource/tasks.py#L1939) | Classification | [WhereIsAI/github-issue-similarity](https://hf.co/datasets/WhereIsAI/github-issue-similarity) |  |
+| 1 | [glue/mnli](src/tasksource/tasks.py#L29) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 2 | [glue/qnli](src/tasksource/tasks.py#L30) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 3 | [glue/rte](src/tasksource/tasks.py#L31) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 4 | [glue/wnli](src/tasksource/tasks.py#L32) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 5 | [glue/mrpc](src/tasksource/tasks.py#L34) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 6 | [glue/qqp](src/tasksource/tasks.py#L35) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 7 | [glue/stsb](src/tasksource/tasks.py#L36) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) | ✓ |
+| 8 | [super_glue/boolq](src/tasksource/tasks.py#L39) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 9 | [super_glue/boolq_passage](src/tasksource/tasks.py#L40) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 10 | [super_glue/cb](src/tasksource/tasks.py#L42) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 11 | [super_glue/multirc](src/tasksource/tasks.py#L43) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 12 | [super_glue/wic](src/tasksource/tasks.py#L48) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 13 | [super_glue/axg](src/tasksource/tasks.py#L53) | Classification | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 14 | [anli/a1](src/tasksource/tasks.py#L56) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
+| 15 | [anli/a2](src/tasksource/tasks.py#L57) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
+| 16 | [anli/a3](src/tasksource/tasks.py#L58) | Classification | [facebook/anli](https://hf.co/datasets/facebook/anli) |  |
+| 17 | [babi_nli/basic-deduction](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 18 | [babi_nli/conjunction](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 19 | [babi_nli/compound-coreference](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 20 | [babi_nli/basic-induction](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 21 | [babi_nli/lists-sets](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 22 | [babi_nli/basic-coreference](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 23 | [babi_nli/path-finding](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 24 | [babi_nli/positional-reasoning](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 25 | [babi_nli/single-supporting-fact](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 26 | [babi_nli/simple-negation](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 27 | [babi_nli/counting](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 28 | [babi_nli/size-reasoning](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 29 | [babi_nli/three-arg-relations](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 30 | [babi_nli/three-supporting-facts](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 31 | [babi_nli/time-reasoning](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 32 | [babi_nli/two-arg-relations](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 33 | [babi_nli/two-supporting-facts](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 34 | [babi_nli/yes-no-questions](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 35 | [babi_nli/indefinite-knowledge](src/tasksource/tasks.py#L61) | Classification | [tasksource/babi_nli](https://hf.co/datasets/tasksource/babi_nli) |  |
+| 36 | [sick/label](src/tasksource/tasks.py#L67) | Classification | [tasksource/sick](https://hf.co/datasets/tasksource/sick) |  |
+| 37 | [sick/relatedness](src/tasksource/tasks.py#L68) | Classification | [tasksource/sick](https://hf.co/datasets/tasksource/sick) | ✓ |
+| 38 | [snli](src/tasksource/tasks.py#L123) | Classification | [stanfordnlp/snli](https://hf.co/datasets/stanfordnlp/snli) |  |
+| 39 | [scitail/snli_format](src/tasksource/tasks.py#L126) | Classification | [allenai/scitail](https://hf.co/datasets/allenai/scitail) |  |
+| 40 | [hans](src/tasksource/tasks.py#L128) | Classification | [tasksource/hans](https://hf.co/datasets/tasksource/hans) |  |
+| 41 | [WANLI](src/tasksource/tasks.py#L131) | Classification | [alisawuffles/WANLI](https://hf.co/datasets/alisawuffles/WANLI) |  |
+| 42 | [recast/recast_megaveridicality](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 43 | [recast/recast_sentiment](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 44 | [recast/recast_ner](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 45 | [recast/recast_verbcorner](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 46 | [recast/recast_verbnet](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 47 | [recast/recast_factuality](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 48 | [recast/recast_puns](src/tasksource/tasks.py#L133) | Classification | [tasksource/recast](https://hf.co/datasets/tasksource/recast) |  |
+| 49 | [probability_words_nli/reasoning_1hop](src/tasksource/tasks.py#L138) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
+| 50 | [probability_words_nli/reasoning_2hop](src/tasksource/tasks.py#L138) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
+| 51 | [probability_words_nli/usnli](src/tasksource/tasks.py#L138) | Classification | [sileod/probability_words_nli](https://hf.co/datasets/sileod/probability_words_nli) |  |
+| 52 | [nan-nli](src/tasksource/tasks.py#L142) | Classification | [joey234/nan-nli](https://hf.co/datasets/joey234/nan-nli) |  |
+| 53 | [nli_fever](src/tasksource/tasks.py#L144) | Classification | [pietrolesci/nli_fever](https://hf.co/datasets/pietrolesci/nli_fever) |  |
+| 54 | [breaking_nli](src/tasksource/tasks.py#L147) | Classification | [pietrolesci/breaking_nli](https://hf.co/datasets/pietrolesci/breaking_nli) |  |
+| 55 | [conj_nli](src/tasksource/tasks.py#L151) | Classification | [pietrolesci/conj_nli](https://hf.co/datasets/pietrolesci/conj_nli) |  |
+| 56 | [fracas](src/tasksource/tasks.py#L155) | Classification | [pietrolesci/fracas](https://hf.co/datasets/pietrolesci/fracas) |  |
+| 57 | [dialogue_nli](src/tasksource/tasks.py#L158) | Classification | [pietrolesci/dialogue_nli](https://hf.co/datasets/pietrolesci/dialogue_nli) |  |
+| 58 | [mpe](src/tasksource/tasks.py#L161) | Classification | [pietrolesci/mpe](https://hf.co/datasets/pietrolesci/mpe) |  |
+| 59 | [dnc](src/tasksource/tasks.py#L165) | Classification | [pietrolesci/dnc](https://hf.co/datasets/pietrolesci/dnc) |  |
+| 60 | [recast_white/fnplus](src/tasksource/tasks.py#L169) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
+| 61 | [recast_white/sprl](src/tasksource/tasks.py#L172) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
+| 62 | [recast_white/dpr](src/tasksource/tasks.py#L175) | Classification | [pietrolesci/recast_white](https://hf.co/datasets/pietrolesci/recast_white) |  |
+| 63 | [joci](src/tasksource/tasks.py#L179) | Classification | [pietrolesci/joci](https://hf.co/datasets/pietrolesci/joci) |  |
+| 64 | [robust_nli/IS_CS](src/tasksource/tasks.py#L185) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 65 | [robust_nli/LI_LI](src/tasksource/tasks.py#L187) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 66 | [robust_nli/ST_WO](src/tasksource/tasks.py#L189) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 67 | [robust_nli/PI_SP](src/tasksource/tasks.py#L191) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 68 | [robust_nli/PI_CD](src/tasksource/tasks.py#L193) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 69 | [robust_nli/ST_SE](src/tasksource/tasks.py#L195) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 70 | [robust_nli/ST_NE](src/tasksource/tasks.py#L197) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 71 | [robust_nli/ST_LM](src/tasksource/tasks.py#L199) | Classification | [pietrolesci/robust_nli](https://hf.co/datasets/pietrolesci/robust_nli) |  |
+| 72 | [robust_nli_is_sd](src/tasksource/tasks.py#L201) | Classification | [pietrolesci/robust_nli_is_sd](https://hf.co/datasets/pietrolesci/robust_nli_is_sd) |  |
+| 73 | [robust_nli_li_ts](src/tasksource/tasks.py#L204) | Classification | [pietrolesci/robust_nli_li_ts](https://hf.co/datasets/pietrolesci/robust_nli_li_ts) |  |
+| 74 | [gen_debiased_nli/snli_seq_z](src/tasksource/tasks.py#L208) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 75 | [gen_debiased_nli/snli_z_aug](src/tasksource/tasks.py#L210) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 76 | [gen_debiased_nli/snli_par_z](src/tasksource/tasks.py#L212) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 77 | [gen_debiased_nli/mnli_par_z](src/tasksource/tasks.py#L214) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 78 | [gen_debiased_nli/mnli_z_aug](src/tasksource/tasks.py#L216) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 79 | [gen_debiased_nli/mnli_seq_z](src/tasksource/tasks.py#L218) | Classification | [pietrolesci/gen_debiased_nli](https://hf.co/datasets/pietrolesci/gen_debiased_nli) |  |
+| 80 | [add_one_rte](src/tasksource/tasks.py#L221) | Classification | [pietrolesci/add_one_rte](https://hf.co/datasets/pietrolesci/add_one_rte) |  |
+| 81 | [hlgd](src/tasksource/tasks.py#L225) | Classification | [tasksource/hlgd](https://hf.co/datasets/tasksource/hlgd) |  |
+| 82 | [paws/labeled_final](src/tasksource/tasks.py#L227) | Classification | [google-research-datasets/paws](https://hf.co/datasets/google-research-datasets/paws) |  |
+| 83 | [paws/labeled_swap](src/tasksource/tasks.py#L228) | Classification | [google-research-datasets/paws](https://hf.co/datasets/google-research-datasets/paws) |  |
+| 84 | [medical_questions_pairs](src/tasksource/tasks.py#L230) | Classification | [curaihealth/medical_questions_pairs](https://hf.co/datasets/curaihealth/medical_questions_pairs) |  |
+| 85 | [conll2003/pos_tags](src/tasksource/tasks.py#L235) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
+| 86 | [conll2003/chunk_tags](src/tasksource/tasks.py#L236) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
+| 87 | [conll2003/ner_tags](src/tasksource/tasks.py#L237) | TokenClassification | [tomaarsen/conll2003](https://hf.co/datasets/tomaarsen/conll2003) |  |
+| 88 | [fig-qa](src/tasksource/tasks.py#L243) | MultipleChoice | [nightingal3/fig-qa](https://hf.co/datasets/nightingal3/fig-qa) |  |
+| 89 | [cos_e/v1.0](src/tasksource/tasks.py#L252) | MultipleChoice | [Salesforce/cos_e](https://hf.co/datasets/Salesforce/cos_e) |  |
+| 90 | [cosmos_qa](src/tasksource/tasks.py#L257) | MultipleChoice | [Samsoup/cosmos_qa](https://hf.co/datasets/Samsoup/cosmos_qa) |  |
+| 91 | [dream](src/tasksource/tasks.py#L260) | MultipleChoice | [dataset-org/dream](https://hf.co/datasets/dataset-org/dream) |  |
+| 92 | [openbookqa](src/tasksource/tasks.py#L267) | MultipleChoice | [allenai/openbookqa](https://hf.co/datasets/allenai/openbookqa) |  |
+| 93 | [qasc](src/tasksource/tasks.py#L273) | MultipleChoice | [allenai/qasc](https://hf.co/datasets/allenai/qasc) |  |
+| 94 | [quartz](src/tasksource/tasks.py#L281) | MultipleChoice | [allenai/quartz](https://hf.co/datasets/allenai/quartz) |  |
+| 95 | [quail](src/tasksource/tasks.py#L286) | MultipleChoice | [textmachinelab/quail](https://hf.co/datasets/textmachinelab/quail) |  |
+| 96 | [head_qa/en](src/tasksource/tasks.py#L292) | MultipleChoice | [EleutherAI/headqa](https://hf.co/datasets/EleutherAI/headqa) |  |
+| 97 | [sciq](src/tasksource/tasks.py#L300) | MultipleChoice | [allenai/sciq](https://hf.co/datasets/allenai/sciq) |  |
+| 98 | [social_i_qa](src/tasksource/tasks.py#L305) | MultipleChoice | [tasksource/social_i_qa](https://hf.co/datasets/tasksource/social_i_qa) |  |
+| 99 | [wiki_hop/original](src/tasksource/tasks.py#L311) | MultipleChoice | [MoE-UNC/wikihop](https://hf.co/datasets/MoE-UNC/wikihop) |  |
+| 100 | [wiqa](src/tasksource/tasks.py#L318) | MultipleChoice | [tasksource/wiqa](https://hf.co/datasets/tasksource/wiqa) |  |
+| 101 | [piqa](src/tasksource/tasks.py#L323) | MultipleChoice | [baber/piqa](https://hf.co/datasets/baber/piqa) |  |
+| 102 | [hellaswag](src/tasksource/tasks.py#L331) | MultipleChoice | [Rowan/hellaswag](https://hf.co/datasets/Rowan/hellaswag) |  |
+| 103 | [super_glue/copa](src/tasksource/tasks.py#L340) | MultipleChoice | [aps/super_glue](https://hf.co/datasets/aps/super_glue) |  |
+| 104 | [balanced-copa](src/tasksource/tasks.py#L342) | MultipleChoice | [pkavumba/balanced-copa](https://hf.co/datasets/pkavumba/balanced-copa) |  |
+| 105 | [e-CARE](src/tasksource/tasks.py#L345) | MultipleChoice | [12ml/e-CARE](https://hf.co/datasets/12ml/e-CARE) |  |
+| 106 | [art](src/tasksource/tasks.py#L348) | MultipleChoice | [allenai/art](https://hf.co/datasets/allenai/art) | ✓ |
+| 107 | [winogrande/winogrande_xl](src/tasksource/tasks.py#L356) | MultipleChoice | [allenai/winogrande](https://hf.co/datasets/allenai/winogrande) |  |
+| 108 | [codah/codah](src/tasksource/tasks.py#L359) | MultipleChoice | [jaredfern/codah](https://hf.co/datasets/jaredfern/codah) |  |
+| 109 | [ai2_arc/ARC-Easy/challenge](src/tasksource/tasks.py#L361) | MultipleChoice | [allenai/ai2_arc](https://hf.co/datasets/allenai/ai2_arc) |  |
+| 110 | [ai2_arc/ARC-Challenge/challenge](src/tasksource/tasks.py#L361) | MultipleChoice | [allenai/ai2_arc](https://hf.co/datasets/allenai/ai2_arc) |  |
+| 111 | [definite_pronoun_resolution](src/tasksource/tasks.py#L366) | MultipleChoice | [community-datasets/definite_pronoun_resolution](https://hf.co/datasets/community-datasets/definite_pronoun_resolution) |  |
+| 112 | [swag/regular](src/tasksource/tasks.py#L372) | MultipleChoice | [allenai/swag](https://hf.co/datasets/allenai/swag) |  |
+| 113 | [math_qa](src/tasksource/tasks.py#L378) | MultipleChoice | [tasksource/math_qa](https://hf.co/datasets/tasksource/math_qa) |  |
+| 114 | [glue/cola](src/tasksource/tasks.py#L387) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 115 | [glue/sst2](src/tasksource/tasks.py#L388) | Classification | [nyu-mll/glue](https://hf.co/datasets/nyu-mll/glue) |  |
+| 116 | [utilitarianism](src/tasksource/tasks.py#L402) | Classification | csv |  |
+| 117 | [amazon_counterfactual/en](src/tasksource/tasks.py#L410) | Classification | [mteb/amazon_counterfactual](https://hf.co/datasets/mteb/amazon_counterfactual) |  |
+| 118 | [insincere-questions](src/tasksource/tasks.py#L415) | Classification | [SetFit/insincere-questions](https://hf.co/datasets/SetFit/insincere-questions) |  |
+| 119 | [toxic_conversations](src/tasksource/tasks.py#L419) | Classification | [SetFit/toxic_conversations](https://hf.co/datasets/SetFit/toxic_conversations) |  |
+| 120 | [TuringBench](src/tasksource/tasks.py#L423) | Classification | csv |  |
+| 121 | [trec](src/tasksource/tasks.py#L432) | Classification | [tasksource/trec](https://hf.co/datasets/tasksource/trec) |  |
+| 122 | [vitaminc](src/tasksource/tasks.py#L435) | Classification | [tals/vitaminc](https://hf.co/datasets/tals/vitaminc) |  |
+| 123 | [hope_edi/english](src/tasksource/tasks.py#L437) | Classification | csv |  |
+| 124 | [rumoureval_2019/RumourEval2019](src/tasksource/tasks.py#L451) | Classification | csv |  |
+| 125 | [ethos/binary](src/tasksource/tasks.py#L465) | Classification | [SetFit/ethos_binary](https://hf.co/datasets/SetFit/ethos_binary) |  |
+| 126 | [ethos/multilabel](src/tasksource/tasks.py#L489) | Classification | [tasksource/ethos](https://hf.co/datasets/tasksource/ethos) |  |
+| 127 | [tweet_eval/emoji](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 128 | [tweet_eval/emotion](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 129 | [tweet_eval/hate](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 130 | [tweet_eval/sentiment](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 131 | [tweet_eval/irony](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 132 | [tweet_eval/offensive](src/tasksource/tasks.py#L492) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) |  |
+| 133 | [tweet_eval/stance_abortion](src/tasksource/tasks.py#L507) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
+| 134 | [tweet_eval/stance_atheism](src/tasksource/tasks.py#L508) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
+| 135 | [tweet_eval/stance_climate](src/tasksource/tasks.py#L509) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
+| 136 | [tweet_eval/stance_feminist](src/tasksource/tasks.py#L510) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
+| 137 | [tweet_eval/stance_hillary](src/tasksource/tasks.py#L511) | Classification | [cardiffnlp/tweet_eval](https://hf.co/datasets/cardiffnlp/tweet_eval) | ✓ |
+| 138 | [discovery/discovery](src/tasksource/tasks.py#L514) | Classification | [sileod/discovery](https://hf.co/datasets/sileod/discovery) |  |
+| 139 | [pragmeval/mrda](src/tasksource/tasks.py#L516) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 140 | [pragmeval/verifiability](src/tasksource/tasks.py#L516) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 141 | [pragmeval/switchboard](src/tasksource/tasks.py#L516) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 142 | [pragmeval/gum](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 143 | [pragmeval/emergent](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 144 | [pragmeval/persuasiveness-premisetype](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 145 | [pragmeval/stac](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 146 | [pragmeval/persuasiveness-claimtype](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 147 | [pragmeval/pdtb](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 148 | [pragmeval/sarcasm](src/tasksource/tasks.py#L520) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 149 | [pragmeval/emobank-arousal](src/tasksource/tasks.py#L529) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 150 | [pragmeval/emobank-dominance](src/tasksource/tasks.py#L530) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 151 | [pragmeval/emobank-valence](src/tasksource/tasks.py#L531) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 152 | [pragmeval/squinky-formality](src/tasksource/tasks.py#L532) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 153 | [pragmeval/squinky-implicature](src/tasksource/tasks.py#L533) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 154 | [pragmeval/squinky-informativeness](src/tasksource/tasks.py#L534) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 155 | [pragmeval/persuasiveness-eloquence](src/tasksource/tasks.py#L535) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 156 | [pragmeval/persuasiveness-relevance](src/tasksource/tasks.py#L536) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 157 | [pragmeval/persuasiveness-specificity](src/tasksource/tasks.py#L537) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 158 | [pragmeval/persuasiveness-strength](src/tasksource/tasks.py#L538) | Classification | [sileod/pragmeval](https://hf.co/datasets/sileod/pragmeval) |  |
+| 159 | [silicone/dyda_da](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 160 | [silicone/dyda_e](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 161 | [silicone/maptask](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 162 | [silicone/oasis](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 163 | [silicone/meld_e](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 164 | [silicone/sem](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 165 | [silicone/meld_s](src/tasksource/tasks.py#L540) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 166 | [silicone/iemocap](src/tasksource/tasks.py#L547) | Classification | [tasksource/silicone](https://hf.co/datasets/tasksource/silicone) |  |
+| 167 | [lex_glue/eurlex](src/tasksource/tasks.py#L552) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
+| 168 | [lex_glue/scotus](src/tasksource/tasks.py#L554) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
+| 169 | [lex_glue/ledgar](src/tasksource/tasks.py#L557) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
+| 170 | [lex_glue/unfair_tos](src/tasksource/tasks.py#L559) | Classification | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) | ✓ |
+| 171 | [lex_glue/case_hold](src/tasksource/tasks.py#L562) | MultipleChoice | [coastalcph/lex_glue](https://hf.co/datasets/coastalcph/lex_glue) |  |
+| 172 | [language-identification](src/tasksource/tasks.py#L570) | Classification | [papluca/language-identification](https://hf.co/datasets/papluca/language-identification) | ✓ |
+| 173 | [imdb](src/tasksource/tasks.py#L575) | Classification | [stanfordnlp/imdb](https://hf.co/datasets/stanfordnlp/imdb) |  |
+| 174 | [rotten_tomatoes](src/tasksource/tasks.py#L577) | Classification | [cornell-movie-review-data/rotten_tomatoes](https://hf.co/datasets/cornell-movie-review-data/rotten_tomatoes) |  |
+| 175 | [ag_news](src/tasksource/tasks.py#L579) | Classification | [fancyzhx/ag_news](https://hf.co/datasets/fancyzhx/ag_news) |  |
+| 176 | [yelp_review_full/yelp_review_full](src/tasksource/tasks.py#L581) | Classification | [Yelp/yelp_review_full](https://hf.co/datasets/Yelp/yelp_review_full) | ✓ |
+| 177 | [financial_phrasebank/sentences_allagree](src/tasksource/tasks.py#L586) | Classification | [ghbacct/financial-phrasebank-all-agree-classification](https://hf.co/datasets/ghbacct/financial-phrasebank-all-agree-classification) |  |
+| 178 | [poem_sentiment](src/tasksource/tasks.py#L591) | Classification | [google-research-datasets/poem_sentiment](https://hf.co/datasets/google-research-datasets/poem_sentiment) |  |
+| 179 | [emotion](src/tasksource/tasks.py#L593) | Classification | [dair-ai/emotion](https://hf.co/datasets/dair-ai/emotion) |  |
+| 180 | [dbpedia_14/dbpedia_14](src/tasksource/tasks.py#L595) | Classification | [fancyzhx/dbpedia_14](https://hf.co/datasets/fancyzhx/dbpedia_14) |  |
+| 181 | [amazon_polarity/amazon_polarity](src/tasksource/tasks.py#L597) | Classification | [fancyzhx/amazon_polarity](https://hf.co/datasets/fancyzhx/amazon_polarity) |  |
+| 182 | [app_reviews](src/tasksource/tasks.py#L599) | Classification | [sealuzh/app_reviews](https://hf.co/datasets/sealuzh/app_reviews) |  |
+| 183 | [hate_speech18](src/tasksource/tasks.py#L603) | Classification | [tasksource/hate_speech18](https://hf.co/datasets/tasksource/hate_speech18) |  |
+| 184 | [sms_spam](src/tasksource/tasks.py#L609) | Classification | [ucirvine/sms_spam](https://hf.co/datasets/ucirvine/sms_spam) |  |
+| 185 | [humicroedit/subtask-1](src/tasksource/tasks.py#L612) | Classification | [tasksource/humicroedit](https://hf.co/datasets/tasksource/humicroedit) | ✓ |
+| 186 | [humicroedit/subtask-2](src/tasksource/tasks.py#L618) | Classification | [tasksource/humicroedit](https://hf.co/datasets/tasksource/humicroedit) | ✓ |
+| 187 | [snips_built_in_intents](src/tasksource/tasks.py#L623) | Classification | [sonos-nlu-benchmark/snips_built_in_intents](https://hf.co/datasets/sonos-nlu-benchmark/snips_built_in_intents) |  |
+| 188 | [hate_speech_offensive](src/tasksource/tasks.py#L627) | Classification | [tdavidson/hate_speech_offensive](https://hf.co/datasets/tdavidson/hate_speech_offensive) |  |
+| 189 | [yahoo_answers_topics](src/tasksource/tasks.py#L629) | Classification | [community-datasets/yahoo_answers_topics](https://hf.co/datasets/community-datasets/yahoo_answers_topics) |  |
+| 190 | [stackoverflow-questions](src/tasksource/tasks.py#L633) | Classification | [pacovaldez/stackoverflow-questions](https://hf.co/datasets/pacovaldez/stackoverflow-questions) | ✓ |
+| 191 | [hyperpartisan_news](src/tasksource/tasks.py#L639) | Classification | [zapsdcn/hyperpartisan_news](https://hf.co/datasets/zapsdcn/hyperpartisan_news) |  |
+| 192 | [sciie](src/tasksource/tasks.py#L644) | Classification | [zapsdcn/sciie](https://hf.co/datasets/zapsdcn/sciie) |  |
+| 193 | [citation_intent](src/tasksource/tasks.py#L645) | Classification | [zapsdcn/citation_intent](https://hf.co/datasets/zapsdcn/citation_intent) |  |
+| 194 | [go_emotions/simplified](src/tasksource/tasks.py#L647) | Classification | [google-research-datasets/go_emotions](https://hf.co/datasets/google-research-datasets/go_emotions) |  |
+| 195 | [scicite](src/tasksource/tasks.py#L651) | Classification | [tasksource/scicite](https://hf.co/datasets/tasksource/scicite) |  |
+| 196 | [liar](src/tasksource/tasks.py#L653) | Classification | [tasksource/liar](https://hf.co/datasets/tasksource/liar) | ✓ |
+| 197 | [lexical_relation_classification/BLESS](src/tasksource/tasks.py#L664) | Classification | json | ✓ |
+| 198 | [lexical_relation_classification/EVALution](src/tasksource/tasks.py#L664) | Classification | json | ✓ |
+| 199 | [lexical_relation_classification/K&H+N](src/tasksource/tasks.py#L664) | Classification | json | ✓ |
+| 200 | [lexical_relation_classification/ROOT09](src/tasksource/tasks.py#L664) | Classification | json | ✓ |
+| 201 | [lexical_relation_classification/CogALexV](src/tasksource/tasks.py#L690) | Classification | json | ✓ |
+| 202 | [linguisticprobing/subj_number](src/tasksource/tasks.py#L708) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 203 | [linguisticprobing/obj_number](src/tasksource/tasks.py#L709) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 204 | [linguisticprobing/past_present](src/tasksource/tasks.py#L710) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 205 | [linguisticprobing/sentence_length](src/tasksource/tasks.py#L711) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 206 | [linguisticprobing/top_constituents](src/tasksource/tasks.py#L712) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 207 | [linguisticprobing/tree_depth](src/tasksource/tasks.py#L714) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 208 | [linguisticprobing/coordination_inversion](src/tasksource/tasks.py#L715) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 209 | [linguisticprobing/odd_man_out](src/tasksource/tasks.py#L717) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 210 | [linguisticprobing/bigram_shift](src/tasksource/tasks.py#L718) | Classification | [tasksource/linguisticprobing](https://hf.co/datasets/tasksource/linguisticprobing) |  |
+| 211 | [crowdflower/political-media-audience](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 212 | [crowdflower/political-media-message](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 213 | [crowdflower/text_emotion](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 214 | [crowdflower/corporate-messaging](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 215 | [crowdflower/economic-news](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 216 | [crowdflower/airline-sentiment](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 217 | [crowdflower/tweet_global_warming](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 218 | [crowdflower/sentiment_nuclear_power](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 219 | [crowdflower/political-media-bias](src/tasksource/tasks.py#L720) | Classification | [tasksource/crowdflower](https://hf.co/datasets/tasksource/crowdflower) |  |
+| 220 | [ethics/commonsense](src/tasksource/tasks.py#L745) | Classification | csv |  |
+| 221 | [ethics/deontology](src/tasksource/tasks.py#L753) | Classification | csv |  |
+| 222 | [ethics/justice](src/tasksource/tasks.py#L761) | Classification | csv |  |
+| 223 | [ethics/virtue](src/tasksource/tasks.py#L769) | Classification | [hendrycks/ethics](https://hf.co/datasets/hendrycks/ethics) |  |
+| 224 | [emo/emo2019](src/tasksource/tasks.py#L778) | Classification | [oneonlee/cleansed_emocontext](https://hf.co/datasets/oneonlee/cleansed_emocontext) |  |
+| 225 | [google_wellformed_query](src/tasksource/tasks.py#L784) | Classification | [tasksource/google_wellformed_query](https://hf.co/datasets/tasksource/google_wellformed_query) | ✓ |
+| 226 | [tweets_hate_speech_detection](src/tasksource/tasks.py#L789) | Classification | [tweets-hate-speech-detection/tweets_hate_speech_detection](https://hf.co/datasets/tweets-hate-speech-detection/tweets_hate_speech_detection) |  |
+| 227 | [wnut_17/wnut_17](src/tasksource/tasks.py#L793) | TokenClassification | [flaitenberger/wnut_17](https://hf.co/datasets/flaitenberger/wnut_17) |  |
+| 228 | [ncbi_disease/ncbi_disease](src/tasksource/tasks.py#L796) | TokenClassification | [ncbi/ncbi_disease](https://hf.co/datasets/ncbi/ncbi_disease) |  |
+| 229 | [acronym_identification](src/tasksource/tasks.py#L799) | TokenClassification | [amirveyseh/acronym_identification](https://hf.co/datasets/amirveyseh/acronym_identification) |  |
+| 230 | [jnlpba/jnlpba](src/tasksource/tasks.py#L802) | TokenClassification | [jnlpba/jnlpba](https://hf.co/datasets/jnlpba/jnlpba) |  |
+| 231 | [ontonotes_english/SpeedOfMagic--ontonotes_english](src/tasksource/tasks.py#L809) | TokenClassification | [SpeedOfMagic/ontonotes_english](https://hf.co/datasets/SpeedOfMagic/ontonotes_english) |  |
+| 232 | [blog_authorship_corpus/gender](src/tasksource/tasks.py#L813) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
+| 233 | [blog_authorship_corpus/age](src/tasksource/tasks.py#L815) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
+| 234 | [blog_authorship_corpus/job](src/tasksource/tasks.py#L818) | Classification | [tasksource/blog_authorship_corpus](https://hf.co/datasets/tasksource/blog_authorship_corpus) | ✓ |
+| 235 | [open_question_type](src/tasksource/tasks.py#L829) | Classification | [Korea-MES/open_question_type](https://hf.co/datasets/Korea-MES/open_question_type) |  |
+| 236 | [health_fact](src/tasksource/tasks.py#L831) | Classification | [marcov/health_fact_promptsource](https://hf.co/datasets/marcov/health_fact_promptsource) |  |
+| 237 | [commonsense_qa](src/tasksource/tasks.py#L835) | MultipleChoice | [tau/commonsense_qa](https://hf.co/datasets/tau/commonsense_qa) |  |
+| 238 | [mc_taco](src/tasksource/tasks.py#L841) | Classification | [marcov/mc_taco_promptsource](https://hf.co/datasets/marcov/mc_taco_promptsource) | ✓ |
+| 239 | [ade_corpus_v2/Ade_corpus_v2_classification](src/tasksource/tasks.py#L848) | Classification | [ade-benchmark-corpus/ade_corpus_v2](https://hf.co/datasets/ade-benchmark-corpus/ade_corpus_v2) |  |
+| 240 | [discosense](src/tasksource/tasks.py#L850) | MultipleChoice | json |  |
+| 241 | [circa](src/tasksource/tasks.py#L857) | Classification | [google-research-datasets/circa](https://hf.co/datasets/google-research-datasets/circa) |  |
+| 242 | [code_x_glue_cc_defect_detection](src/tasksource/tasks.py#L862) | Classification | [google/code_x_glue_cc_defect_detection](https://hf.co/datasets/google/code_x_glue_cc_defect_detection) |  |
+| 243 | [phrase_similarity](src/tasksource/tasks.py#L866) | Classification | [Deehan1866/processed_phrase_similarity](https://hf.co/datasets/Deehan1866/processed_phrase_similarity) |  |
+| 244 | [scientific-exaggeration-detection](src/tasksource/tasks.py#L874) | Classification | [copenlu/scientific-exaggeration-detection](https://hf.co/datasets/copenlu/scientific-exaggeration-detection) |  |
+| 245 | [quarel](src/tasksource/tasks.py#L880) | Classification | [community-datasets/quarel](https://hf.co/datasets/community-datasets/quarel) |  |
+| 246 | [fever-evidence-related](src/tasksource/tasks.py#L885) | Classification | [mwong/fever-evidence-related](https://hf.co/datasets/mwong/fever-evidence-related) |  |
+| 247 | [numer_sense](src/tasksource/tasks.py#L888) | Classification | [tasksource/numer_sense](https://hf.co/datasets/tasksource/numer_sense) |  |
+| 248 | [dynasent/dynabench.dynasent.r1.all/r1](src/tasksource/tasks.py#L895) | Classification | [tasksource/dynasent](https://hf.co/datasets/tasksource/dynasent) |  |
+| 249 | [dynasent/dynabench.dynasent.r2.all/r2](src/tasksource/tasks.py#L899) | Classification | [tasksource/dynasent](https://hf.co/datasets/tasksource/dynasent) |  |
+| 250 | [Sarcasm_News_Headline](src/tasksource/tasks.py#L904) | Classification | [raquiba/Sarcasm_News_Headline](https://hf.co/datasets/raquiba/Sarcasm_News_Headline) |  |
+| 251 | [sem_eval_2010_task_8](src/tasksource/tasks.py#L907) | Classification | [SemEvalWorkshop/sem_eval_2010_task_8](https://hf.co/datasets/SemEvalWorkshop/sem_eval_2010_task_8) |  |
+| 252 | [auditor_review](src/tasksource/tasks.py#L909) | Classification | [demo-org/auditor_review](https://hf.co/datasets/demo-org/auditor_review) |  |
+| 253 | [medmcqa](src/tasksource/tasks.py#L913) | MultipleChoice | [openlifescienceai/medmcqa](https://hf.co/datasets/openlifescienceai/medmcqa) |  |
+| 254 | [Dynasent_Disagreement](src/tasksource/tasks.py#L930) | Classification | [RuyuanWan/Dynasent_Disagreement](https://hf.co/datasets/RuyuanWan/Dynasent_Disagreement) | ✓ |
+| 255 | [Politeness_Disagreement](src/tasksource/tasks.py#L932) | Classification | [RuyuanWan/Politeness_Disagreement](https://hf.co/datasets/RuyuanWan/Politeness_Disagreement) | ✓ |
+| 256 | [SBIC_Disagreement](src/tasksource/tasks.py#L934) | Classification | [RuyuanWan/SBIC_Disagreement](https://hf.co/datasets/RuyuanWan/SBIC_Disagreement) | ✓ |
+| 257 | [SChem_Disagreement](src/tasksource/tasks.py#L936) | Classification | [RuyuanWan/SChem_Disagreement](https://hf.co/datasets/RuyuanWan/SChem_Disagreement) | ✓ |
+| 258 | [Dilemmas_Disagreement](src/tasksource/tasks.py#L938) | Classification | [RuyuanWan/Dilemmas_Disagreement](https://hf.co/datasets/RuyuanWan/Dilemmas_Disagreement) | ✓ |
+| 259 | [logiqa](src/tasksource/tasks.py#L941) | MultipleChoice | [fireworks-ai/logiqa](https://hf.co/datasets/fireworks-ai/logiqa) |  |
+| 260 | [wiki_qa](src/tasksource/tasks.py#L950) | Classification | [microsoft/wiki_qa](https://hf.co/datasets/microsoft/wiki_qa) | ✓ |
+| 261 | [cycic_classification](src/tasksource/tasks.py#L952) | Classification | [tasksource/cycic_classification](https://hf.co/datasets/tasksource/cycic_classification) |  |
+| 262 | [cycic_multiplechoice](src/tasksource/tasks.py#L954) | MultipleChoice | [tasksource/cycic_multiplechoice](https://hf.co/datasets/tasksource/cycic_multiplechoice) |  |
+| 263 | [sts-companion](src/tasksource/tasks.py#L958) | Classification | [tasksource/sts-companion](https://hf.co/datasets/tasksource/sts-companion) |  |
+| 264 | [commonsense_qa_2.0](src/tasksource/tasks.py#L961) | Classification | [tasksource/commonsense_qa_2.0](https://hf.co/datasets/tasksource/commonsense_qa_2.0) |  |
+| 265 | [lingnli](src/tasksource/tasks.py#L964) | Classification | [tasksource/lingnli](https://hf.co/datasets/tasksource/lingnli) |  |
+| 266 | [monotonicity-entailment](src/tasksource/tasks.py#L966) | Classification | [tasksource/monotonicity-entailment](https://hf.co/datasets/tasksource/monotonicity-entailment) |  |
+| 267 | [arct](src/tasksource/tasks.py#L969) | MultipleChoice | [tasksource/arct](https://hf.co/datasets/tasksource/arct) |  |
+| 268 | [scinli](src/tasksource/tasks.py#L972) | Classification | [tasksource/scinli](https://hf.co/datasets/tasksource/scinli) |  |
+| 269 | [naturallogic](src/tasksource/tasks.py#L976) | Classification | [tasksource/naturallogic](https://hf.co/datasets/tasksource/naturallogic) |  |
+| 270 | [onestop_qa](src/tasksource/tasks.py#L978) | MultipleChoice | [malmaud/onestop_qa](https://hf.co/datasets/malmaud/onestop_qa) |  |
+| 271 | [moral_stories/full](src/tasksource/tasks.py#L981) | MultipleChoice | [LabHC/moral_stories](https://hf.co/datasets/LabHC/moral_stories) |  |
+| 272 | [prost](src/tasksource/tasks.py#L989) | MultipleChoice | json |  |
+| 273 | [dynahate](src/tasksource/tasks.py#L994) | Classification | [tasksource/dynahate](https://hf.co/datasets/tasksource/dynahate) |  |
+| 274 | [syntactic-augmentation-nli](src/tasksource/tasks.py#L996) | Classification | [tasksource/syntactic-augmentation-nli](https://hf.co/datasets/tasksource/syntactic-augmentation-nli) |  |
+| 275 | [autotnli](src/tasksource/tasks.py#L998) | Classification | [tasksource/autotnli](https://hf.co/datasets/tasksource/autotnli) |  |
+| 276 | [CONDAQA](src/tasksource/tasks.py#L1000) | Classification | [lasha-nlp/CONDAQA](https://hf.co/datasets/lasha-nlp/CONDAQA) |  |
+| 277 | [webgpt_comparisons](src/tasksource/tasks.py#L1010) | MultipleChoice | [heegyu/webgpt_comparisons_ko](https://hf.co/datasets/heegyu/webgpt_comparisons_ko) | ✓ |
+| 278 | [synthetic-instruct-gptj-pairwise](src/tasksource/tasks.py#L1018) | MultipleChoice | [Dahoas/synthetic-instruct-gptj-pairwise](https://hf.co/datasets/Dahoas/synthetic-instruct-gptj-pairwise) | ✓ |
+| 279 | [scruples](src/tasksource/tasks.py#L1021) | Classification | [tasksource/scruples](https://hf.co/datasets/tasksource/scruples) | ✓ |
+| 280 | [wouldyourather](src/tasksource/tasks.py#L1023) | MultipleChoice | [tasksource/wouldyourather](https://hf.co/datasets/tasksource/wouldyourather) | ✓ |
+| 281 | [defeasible-nli/atomic](src/tasksource/tasks.py#L1031) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
+| 282 | [defeasible-nli/snli](src/tasksource/tasks.py#L1031) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
+| 283 | [defeasible-nli/social](src/tasksource/tasks.py#L1034) | Classification | [tasksource/defeasible-nli](https://hf.co/datasets/tasksource/defeasible-nli) |  |
+| 284 | [help-nli](src/tasksource/tasks.py#L1037) | Classification | [tasksource/help-nli](https://hf.co/datasets/tasksource/help-nli) |  |
+| 285 | [nli-veridicality-transitivity](src/tasksource/tasks.py#L1040) | Classification | [tasksource/nli-veridicality-transitivity](https://hf.co/datasets/tasksource/nli-veridicality-transitivity) |  |
+| 286 | [lonli](src/tasksource/tasks.py#L1043) | Classification | [tasksource/lonli](https://hf.co/datasets/tasksource/lonli) |  |
+| 287 | [dadc-limit-nli](src/tasksource/tasks.py#L1046) | Classification | [tasksource/dadc-limit-nli](https://hf.co/datasets/tasksource/dadc-limit-nli) |  |
+| 288 | [FLUTE](src/tasksource/tasks.py#L1049) | Classification | [ColumbiaNLP/FLUTE](https://hf.co/datasets/ColumbiaNLP/FLUTE) |  |
+| 289 | [strategy-qa](src/tasksource/tasks.py#L1052) | Classification | [tasksource/strategy-qa](https://hf.co/datasets/tasksource/strategy-qa) |  |
+| 290 | [summarize_from_feedback/comparisons](src/tasksource/tasks.py#L1055) | MultipleChoice | [vwxyzjn/summarize_from_feedback_oai_preprocessing](https://hf.co/datasets/vwxyzjn/summarize_from_feedback_oai_preprocessing) | ✓ |
+| 291 | [folio](src/tasksource/tasks.py#L1063) | Classification | [tasksource/folio](https://hf.co/datasets/tasksource/folio) |  |
+| 292 | [tomi-nli](src/tasksource/tasks.py#L1067) | Classification | [tasksource/tomi-nli](https://hf.co/datasets/tasksource/tomi-nli) |  |
+| 293 | [avicenna](src/tasksource/tasks.py#L1070) | Classification | [tasksource/avicenna](https://hf.co/datasets/tasksource/avicenna) | ✓ |
+| 294 | [SHP](src/tasksource/tasks.py#L1073) | MultipleChoice | [stanfordnlp/SHP](https://hf.co/datasets/stanfordnlp/SHP) | ✓ |
+| 295 | [MedQA-USMLE-4-options-hf](src/tasksource/tasks.py#L1081) | MultipleChoice | [GBaker/MedQA-USMLE-4-options-hf](https://hf.co/datasets/GBaker/MedQA-USMLE-4-options-hf) |  |
+| 296 | [wikimedqa/medwiki](src/tasksource/tasks.py#L1084) | MultipleChoice | [sileod/wikimedqa](https://hf.co/datasets/sileod/wikimedqa) |  |
+| 297 | [cicero](src/tasksource/tasks.py#L1093) | MultipleChoice | [declare-lab/cicero](https://hf.co/datasets/declare-lab/cicero) |  |
+| 298 | [CREAK](src/tasksource/tasks.py#L1097) | Classification | [amydeng2000/CREAK](https://hf.co/datasets/amydeng2000/CREAK) |  |
+| 299 | [mutual](src/tasksource/tasks.py#L1100) | MultipleChoice | [tasksource/mutual](https://hf.co/datasets/tasksource/mutual) |  |
+| 300 | [puzzte](src/tasksource/tasks.py#L1104) | Classification | [tasksource/puzzte](https://hf.co/datasets/tasksource/puzzte) |  |
+| 301 | [implicatures](src/tasksource/tasks.py#L1109) | MultipleChoice | [tasksource/implicatures](https://hf.co/datasets/tasksource/implicatures) |  |
+| 302 | [race/high](src/tasksource/tasks.py#L1114) | MultipleChoice | [ehovy/race](https://hf.co/datasets/ehovy/race) |  |
+| 303 | [race/middle](src/tasksource/tasks.py#L1114) | MultipleChoice | [ehovy/race](https://hf.co/datasets/ehovy/race) |  |
+| 304 | [race-c](src/tasksource/tasks.py#L1118) | MultipleChoice | [tasksource/race-c](https://hf.co/datasets/tasksource/race-c) |  |
+| 305 | [spartqa-yn](src/tasksource/tasks.py#L1121) | Classification | [tasksource/spartqa-yn](https://hf.co/datasets/tasksource/spartqa-yn) |  |
+| 306 | [spartqa-mchoice](src/tasksource/tasks.py#L1124) | MultipleChoice | [tasksource/spartqa-mchoice](https://hf.co/datasets/tasksource/spartqa-mchoice) |  |
+| 307 | [temporal-nli](src/tasksource/tasks.py#L1127) | Classification | [tasksource/temporal-nli](https://hf.co/datasets/tasksource/temporal-nli) |  |
+| 308 | [riddle_sense](src/tasksource/tasks.py#L1130) | MultipleChoice | [jeggers/riddle_sense](https://hf.co/datasets/jeggers/riddle_sense) |  |
+| 309 | [clcd-english](src/tasksource/tasks.py#L1135) | Classification | [tasksource/clcd-english](https://hf.co/datasets/tasksource/clcd-english) |  |
+| 310 | [twentyquestions](src/tasksource/tasks.py#L1147) | Classification | [tasksource/twentyquestions](https://hf.co/datasets/tasksource/twentyquestions) |  |
+| 311 | [reclor](src/tasksource/tasks.py#L1152) | MultipleChoice | [tasksource/reclor](https://hf.co/datasets/tasksource/reclor) |  |
+| 312 | [counterfactually-augmented-imdb](src/tasksource/tasks.py#L1155) | Classification | [tasksource/counterfactually-augmented-imdb](https://hf.co/datasets/tasksource/counterfactually-augmented-imdb) |  |
+| 313 | [counterfactually-augmented-snli](src/tasksource/tasks.py#L1158) | Classification | [tasksource/counterfactually-augmented-snli](https://hf.co/datasets/tasksource/counterfactually-augmented-snli) |  |
+| 314 | [cnli](src/tasksource/tasks.py#L1161) | Classification | [tasksource/cnli](https://hf.co/datasets/tasksource/cnli) |  |
+| 315 | [boolq-natural-perturbations](src/tasksource/tasks.py#L1164) | Classification | [tasksource/boolq-natural-perturbations](https://hf.co/datasets/tasksource/boolq-natural-perturbations) |  |
+| 316 | [acceptability-prediction](src/tasksource/tasks.py#L1168) | Classification | [tasksource/acceptability-prediction](https://hf.co/datasets/tasksource/acceptability-prediction) | ✓ |
+| 317 | [equate](src/tasksource/tasks.py#L1172) | Classification | [tasksource/equate](https://hf.co/datasets/tasksource/equate) |  |
+| 318 | [ScienceQA_text_only](src/tasksource/tasks.py#L1175) | MultipleChoice | [tasksource/ScienceQA_text_only](https://hf.co/datasets/tasksource/ScienceQA_text_only) |  |
+| 319 | [ekar_english](src/tasksource/tasks.py#L1178) | MultipleChoice | [Jiangjie/ekar_english](https://hf.co/datasets/Jiangjie/ekar_english) | ✓ |
+| 320 | [implicit-hate-stg1](src/tasksource/tasks.py#L1182) | Classification | [tasksource/implicit-hate-stg1](https://hf.co/datasets/tasksource/implicit-hate-stg1) |  |
+| 321 | [chaos-mnli-ambiguity](src/tasksource/tasks.py#L1185) | Classification | [tasksource/chaos-mnli-ambiguity](https://hf.co/datasets/tasksource/chaos-mnli-ambiguity) | ✓ |
+| 322 | [headline_cause/en_simple](src/tasksource/tasks.py#L1189) | Classification | json |  |
+| 323 | [logiqa-2.0-nli](src/tasksource/tasks.py#L1194) | Classification | [tasksource/logiqa-2.0-nli](https://hf.co/datasets/tasksource/logiqa-2.0-nli) |  |
+| 324 | [oasst2_dense_flat/quality](src/tasksource/tasks.py#L1199) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
+| 325 | [oasst2_dense_flat/toxicity](src/tasksource/tasks.py#L1201) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
+| 326 | [oasst2_dense_flat/helpfulness](src/tasksource/tasks.py#L1203) | Classification | [tasksource/oasst2_dense_flat](https://hf.co/datasets/tasksource/oasst2_dense_flat) | ✓ |
+| 327 | [mindgames](src/tasksource/tasks.py#L1206) | Classification | [sileod/mindgames](https://hf.co/datasets/sileod/mindgames) |  |
+| 328 | [universal_dependencies/en_gum/deprel](src/tasksource/tasks.py#L1220) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
+| 329 | [universal_dependencies/en_partut/deprel](src/tasksource/tasks.py#L1220) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
+| 330 | [universal_dependencies/en_ewt/deprel](src/tasksource/tasks.py#L1220) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
+| 331 | [universal_dependencies/en_lines/deprel](src/tasksource/tasks.py#L1220) | TokenClassification | [universal-dependencies/universal_dependencies](https://hf.co/datasets/universal-dependencies/universal_dependencies) |  |
+| 332 | [ambient](src/tasksource/tasks.py#L1226) | Classification | [tasksource/ambient](https://hf.co/datasets/tasksource/ambient) | ✓ |
+| 333 | [path-naturalness-prediction](src/tasksource/tasks.py#L1229) | MultipleChoice | [tasksource/path-naturalness-prediction](https://hf.co/datasets/tasksource/path-naturalness-prediction) | ✓ |
+| 334 | [civil_comments/toxicity](src/tasksource/tasks.py#L1239) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 335 | [civil_comments/severe_toxicity](src/tasksource/tasks.py#L1240) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 336 | [civil_comments/obscene](src/tasksource/tasks.py#L1241) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 337 | [civil_comments/threat](src/tasksource/tasks.py#L1242) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 338 | [civil_comments/insult](src/tasksource/tasks.py#L1243) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 339 | [civil_comments/identity_attack](src/tasksource/tasks.py#L1244) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 340 | [civil_comments/sexual_explicit](src/tasksource/tasks.py#L1245) | Classification | [google/civil_comments](https://hf.co/datasets/google/civil_comments) | ✓ |
+| 341 | [cloth](src/tasksource/tasks.py#L1247) | MultipleChoice | [AndyChiang/cloth](https://hf.co/datasets/AndyChiang/cloth) |  |
+| 342 | [dgen](src/tasksource/tasks.py#L1248) | MultipleChoice | [AndyChiang/dgen](https://hf.co/datasets/AndyChiang/dgen) |  |
+| 343 | [I2D2](src/tasksource/tasks.py#L1250) | Classification | [tasksource/I2D2](https://hf.co/datasets/tasksource/I2D2) |  |
+| 344 | [args_me](src/tasksource/tasks.py#L1252) | Classification | [webis/args_me](https://hf.co/datasets/webis/args_me) |  |
+| 345 | [Touche23-ValueEval](src/tasksource/tasks.py#L1255) | Classification | csv |  |
+| 346 | [starcon](src/tasksource/tasks.py#L1263) | Classification | [tasksource/starcon](https://hf.co/datasets/tasksource/starcon) |  |
+| 347 | [banking77](src/tasksource/tasks.py#L1265) | Classification | [legacy-datasets/banking77](https://hf.co/datasets/legacy-datasets/banking77) |  |
+| 348 | [it-support-tickets](src/tasksource/tasks.py#L1267) | Classification | [tasksource/it-support-tickets](https://hf.co/datasets/tasksource/it-support-tickets) |  |
+| 349 | [ConTRoL-nli](src/tasksource/tasks.py#L1271) | Classification | [tasksource/ConTRoL-nli](https://hf.co/datasets/tasksource/ConTRoL-nli) |  |
+| 350 | [tracie](src/tasksource/tasks.py#L1272) | Classification | [tasksource/tracie](https://hf.co/datasets/tasksource/tracie) |  |
+| 351 | [sherliic](src/tasksource/tasks.py#L1273) | Classification | [tasksource/sherliic](https://hf.co/datasets/tasksource/sherliic) |  |
+| 352 | [sen-making/1](src/tasksource/tasks.py#L1275) | MultipleChoice | [tasksource/sen-making](https://hf.co/datasets/tasksource/sen-making) | ✓ |
+| 353 | [sen-making/2](src/tasksource/tasks.py#L1279) | MultipleChoice | [tasksource/sen-making](https://hf.co/datasets/tasksource/sen-making) | ✓ |
+| 354 | [winowhy](src/tasksource/tasks.py#L1282) | Classification | [tasksource/winowhy](https://hf.co/datasets/tasksource/winowhy) | ✓ |
+| 355 | [robustLR](src/tasksource/tasks.py#L1286) | Classification | [tasksource/robustLR](https://hf.co/datasets/tasksource/robustLR) |  |
+| 356 | [clutrr](src/tasksource/tasks.py#L1288) | Classification | [tasksource/clutrr](https://hf.co/datasets/tasksource/clutrr) |  |
+| 357 | [logical-fallacy](src/tasksource/tasks.py#L1290) | Classification | [tasksource/logical-fallacy](https://hf.co/datasets/tasksource/logical-fallacy) |  |
+| 358 | [parade](src/tasksource/tasks.py#L1292) | Classification | [tasksource/parade](https://hf.co/datasets/tasksource/parade) |  |
+| 359 | [cladder](src/tasksource/tasks.py#L1294) | Classification | [tasksource/cladder](https://hf.co/datasets/tasksource/cladder) |  |
+| 360 | [subjectivity](src/tasksource/tasks.py#L1296) | Classification | [tasksource/subjectivity](https://hf.co/datasets/tasksource/subjectivity) |  |
+| 361 | [MOH](src/tasksource/tasks.py#L1298) | Classification | [tasksource/MOH](https://hf.co/datasets/tasksource/MOH) |  |
+| 362 | [VUAC](src/tasksource/tasks.py#L1299) | Classification | [tasksource/VUAC](https://hf.co/datasets/tasksource/VUAC) |  |
+| 363 | [TroFi](src/tasksource/tasks.py#L1300) | Classification | parquet |  |
+| 364 | [sharc](src/tasksource/tasks.py#L1307) | Classification | [tasksource/sharc](https://hf.co/datasets/tasksource/sharc) |  |
+| 365 | [conceptrules_v2](src/tasksource/tasks.py#L1311) | Classification | [tasksource/conceptrules_v2](https://hf.co/datasets/tasksource/conceptrules_v2) | ✓ |
+| 366 | [disrpt/eng.dep.scidtb.rels](src/tasksource/tasks.py#L1313) | Classification | [multilingual-discourse-hub/disrpt](https://hf.co/datasets/multilingual-discourse-hub/disrpt) |  |
+| 367 | [conll2000](src/tasksource/tasks.py#L1315) | TokenClassification | [eriktks/conll2000](https://hf.co/datasets/eriktks/conll2000) |  |
+| 368 | [few-nerd/supervised](src/tasksource/tasks.py#L1318) | TokenClassification | [DFKI-SLT/few-nerd](https://hf.co/datasets/DFKI-SLT/few-nerd) |  |
+| 369 | [finer-139](src/tasksource/tasks.py#L1319) | TokenClassification | [nlpaueb/finer-139](https://hf.co/datasets/nlpaueb/finer-139) |  |
+| 370 | [zero-shot-label-nli](src/tasksource/tasks.py#L1322) | Classification | [tasksource/zero-shot-label-nli](https://hf.co/datasets/tasksource/zero-shot-label-nli) |  |
+| 371 | [com2sense](src/tasksource/tasks.py#L1324) | Classification | [tasksource/com2sense](https://hf.co/datasets/tasksource/com2sense) |  |
+| 372 | [scone](src/tasksource/tasks.py#L1326) | Classification | [tasksource/scone](https://hf.co/datasets/tasksource/scone) |  |
+| 373 | [winodict](src/tasksource/tasks.py#L1328) | MultipleChoice | [tasksource/winodict](https://hf.co/datasets/tasksource/winodict) |  |
+| 374 | [fool-me-twice](src/tasksource/tasks.py#L1330) | Classification | [tasksource/fool-me-twice](https://hf.co/datasets/tasksource/fool-me-twice) |  |
+| 375 | [monli](src/tasksource/tasks.py#L1334) | Classification | [tasksource/monli](https://hf.co/datasets/tasksource/monli) |  |
+| 376 | [corr2cause](src/tasksource/tasks.py#L1336) | Classification | [tasksource/corr2cause](https://hf.co/datasets/tasksource/corr2cause) |  |
+| 377 | [lsat_qa/all](src/tasksource/tasks.py#L1338) | MultipleChoice | [lighteval/lsat_qa](https://hf.co/datasets/lighteval/lsat_qa) |  |
+| 378 | [apt](src/tasksource/tasks.py#L1340) | Classification | [tasksource/apt](https://hf.co/datasets/tasksource/apt) |  |
+| 379 | [twitter-financial-news-sentiment](src/tasksource/tasks.py#L1343) | Classification | [zeroshot/twitter-financial-news-sentiment](https://hf.co/datasets/zeroshot/twitter-financial-news-sentiment) |  |
+| 380 | [icl-symbol-tuning-instruct](src/tasksource/tasks.py#L1350) | Classification | [tasksource/icl-symbol-tuning-instruct](https://hf.co/datasets/tasksource/icl-symbol-tuning-instruct) | ✓ |
+| 381 | [SpaceNLI](src/tasksource/tasks.py#L1356) | Classification | [tasksource/SpaceNLI](https://hf.co/datasets/tasksource/SpaceNLI) |  |
+| 382 | [propsegment/nli](src/tasksource/tasks.py#L1358) | Classification | json |  |
+| 383 | [HatemojiBuild](src/tasksource/tasks.py#L1367) | Classification | [HannahRoseKirk/HatemojiBuild](https://hf.co/datasets/HannahRoseKirk/HatemojiBuild) |  |
+| 384 | [regset](src/tasksource/tasks.py#L1370) | Classification | [tasksource/regset](https://hf.co/datasets/tasksource/regset) | ✓ |
+| 385 | [esci](src/tasksource/tasks.py#L1376) | Classification | [tasksource/esci](https://hf.co/datasets/tasksource/esci) |  |
+| 386 | [chatbot_arena_conversations](src/tasksource/tasks.py#L1395) | MultipleChoice | [lmsys/chatbot_arena_conversations](https://hf.co/datasets/lmsys/chatbot_arena_conversations) | ✓ |
+| 387 | [dnd_style_intents](src/tasksource/tasks.py#L1401) | Classification | [neurae/dnd_style_intents](https://hf.co/datasets/neurae/dnd_style_intents) |  |
+| 388 | [FLD.v2/default](src/tasksource/tasks.py#L1404) | Classification | [hitachi-nlp/FLD.v2](https://hf.co/datasets/hitachi-nlp/FLD.v2) |  |
+| 389 | [FLD.v2/star](src/tasksource/tasks.py#L1407) | Classification | [hitachi-nlp/FLD.v2](https://hf.co/datasets/hitachi-nlp/FLD.v2) |  |
+| 390 | [SDOH-NLI](src/tasksource/tasks.py#L1410) | Classification | [tasksource/SDOH-NLI](https://hf.co/datasets/tasksource/SDOH-NLI) |  |
+| 391 | [scifact_entailment](src/tasksource/tasks.py#L1413) | Classification | [tasksource/scifact_entailment](https://hf.co/datasets/tasksource/scifact_entailment) |  |
+| 392 | [feasibilityQA](src/tasksource/tasks.py#L1417) | Classification | [tasksource/feasibilityQA](https://hf.co/datasets/tasksource/feasibilityQA) |  |
+| 393 | [simple_pair](src/tasksource/tasks.py#L1420) | Classification | [tasksource/simple_pair](https://hf.co/datasets/tasksource/simple_pair) |  |
+| 394 | [AdjectiveScaleProbe-nli](src/tasksource/tasks.py#L1421) | Classification | [tasksource/AdjectiveScaleProbe-nli](https://hf.co/datasets/tasksource/AdjectiveScaleProbe-nli) |  |
+| 395 | [resnli](src/tasksource/tasks.py#L1422) | Classification | [tasksource/resnli](https://hf.co/datasets/tasksource/resnli) |  |
+| 396 | [SpaRTUN](src/tasksource/tasks.py#L1424) | MultipleChoice | [tasksource/SpaRTUN](https://hf.co/datasets/tasksource/SpaRTUN) |  |
+| 397 | [ReSQ](src/tasksource/tasks.py#L1429) | MultipleChoice | [tasksource/ReSQ](https://hf.co/datasets/tasksource/ReSQ) |  |
+| 398 | [semantic_fragments_nli](src/tasksource/tasks.py#L1434) | Classification | [tasksource/semantic_fragments_nli](https://hf.co/datasets/tasksource/semantic_fragments_nli) |  |
+| 399 | [dataset_train_nli](src/tasksource/tasks.py#L1437) | Classification | [MoritzLaurer/dataset_train_nli](https://hf.co/datasets/MoritzLaurer/dataset_train_nli) |  |
+| 400 | [stepgame](src/tasksource/tasks.py#L1442) | Classification | [tasksource/stepgame](https://hf.co/datasets/tasksource/stepgame) |  |
+| 401 | [nlgraph](src/tasksource/tasks.py#L1450) | Classification | [tasksource/nlgraph](https://hf.co/datasets/tasksource/nlgraph) |  |
+| 402 | [oasst2_pairwise_rlhf_reward](src/tasksource/tasks.py#L1454) | MultipleChoice | [tasksource/oasst2_pairwise_rlhf_reward](https://hf.co/datasets/tasksource/oasst2_pairwise_rlhf_reward) | ✓ |
+| 403 | [hh-rlhf/helpful-rejection-sampled](src/tasksource/tasks.py#L1465) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
+| 404 | [hh-rlhf/helpful-online](src/tasksource/tasks.py#L1465) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
+| 405 | [hh-rlhf/helpful-base](src/tasksource/tasks.py#L1465) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
+| 406 | [hh-rlhf/harmless-base](src/tasksource/tasks.py#L1469) | MultipleChoice | [tasksource/hh-rlhf](https://hf.co/datasets/tasksource/hh-rlhf) | ✓ |
+| 407 | [ruletaker](src/tasksource/tasks.py#L1473) | Classification | [tasksource/ruletaker](https://hf.co/datasets/tasksource/ruletaker) | ✓ |
+| 408 | [PARARULE-Plus](src/tasksource/tasks.py#L1477) | Classification | [qbao775/PARARULE-Plus](https://hf.co/datasets/qbao775/PARARULE-Plus) | ✓ |
+| 409 | [proofwriter](src/tasksource/tasks.py#L1481) | Classification | [tasksource/proofwriter](https://hf.co/datasets/tasksource/proofwriter) |  |
+| 410 | [logical-entailment](src/tasksource/tasks.py#L1484) | Classification | [tasksource/logical-entailment](https://hf.co/datasets/tasksource/logical-entailment) |  |
+| 411 | [nope](src/tasksource/tasks.py#L1486) | Classification | [tasksource/nope](https://hf.co/datasets/tasksource/nope) |  |
+| 412 | [LogicNLI](src/tasksource/tasks.py#L1490) | Classification | [tasksource/LogicNLI](https://hf.co/datasets/tasksource/LogicNLI) |  |
+| 413 | [contract-nli/contractnli_a/seg](src/tasksource/tasks.py#L1492) | Classification | [tasksource/contract-nli](https://hf.co/datasets/tasksource/contract-nli) |  |
+| 414 | [contract-nli/contractnli_b/full](src/tasksource/tasks.py#L1494) | Classification | [tasksource/contract-nli](https://hf.co/datasets/tasksource/contract-nli) |  |
+| 415 | [nli4ct_semeval2024](src/tasksource/tasks.py#L1496) | Classification | [AshtonIsNotHere/nli4ct_semeval2024](https://hf.co/datasets/AshtonIsNotHere/nli4ct_semeval2024) |  |
+| 416 | [lsat-ar](src/tasksource/tasks.py#L1499) | MultipleChoice | [tasksource/lsat-ar](https://hf.co/datasets/tasksource/lsat-ar) |  |
+| 417 | [lsat-rc](src/tasksource/tasks.py#L1504) | MultipleChoice | [tasksource/lsat-rc](https://hf.co/datasets/tasksource/lsat-rc) |  |
+| 418 | [biosift-nli](src/tasksource/tasks.py#L1509) | Classification | [AshtonIsNotHere/biosift-nli](https://hf.co/datasets/AshtonIsNotHere/biosift-nli) |  |
+| 419 | [brainteasers/SP](src/tasksource/tasks.py#L1513) | MultipleChoice | [tasksource/brainteasers](https://hf.co/datasets/tasksource/brainteasers) |  |
+| 420 | [brainteasers/WP](src/tasksource/tasks.py#L1513) | MultipleChoice | [tasksource/brainteasers](https://hf.co/datasets/tasksource/brainteasers) |  |
+| 421 | [toxigen-data/annotated](src/tasksource/tasks.py#L1519) | Classification | [skg/toxigen-data](https://hf.co/datasets/skg/toxigen-data) |  |
+| 422 | [persuasion](src/tasksource/tasks.py#L1530) | Classification | [Anthropic/persuasion](https://hf.co/datasets/Anthropic/persuasion) |  |
+| 423 | [AmbigNQ-clarifying-question](src/tasksource/tasks.py#L1536) | Classification | [erbacher/AmbigNQ-clarifying-question](https://hf.co/datasets/erbacher/AmbigNQ-clarifying-question) |  |
+| 424 | [SIGA-nli](src/tasksource/tasks.py#L1539) | Classification | [tasksource/SIGA-nli](https://hf.co/datasets/tasksource/SIGA-nli) |  |
+| 425 | [FOL-nli](src/tasksource/tasks.py#L1541) | Classification | [unigram/FOL-nli](https://hf.co/datasets/unigram/FOL-nli) |  |
+| 426 | [goal-step-wikihow/goal](src/tasksource/tasks.py#L1543) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
+| 427 | [goal-step-wikihow/step](src/tasksource/tasks.py#L1546) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
+| 428 | [goal-step-wikihow/order](src/tasksource/tasks.py#L1549) | MultipleChoice | [tasksource/goal-step-wikihow](https://hf.co/datasets/tasksource/goal-step-wikihow) |  |
+| 429 | [PARADISE](src/tasksource/tasks.py#L1552) | MultipleChoice | [GGLab/PARADISE](https://hf.co/datasets/GGLab/PARADISE) |  |
+| 430 | [doc-nli](src/tasksource/tasks.py#L1555) | Classification | [tasksource/doc-nli](https://hf.co/datasets/tasksource/doc-nli) |  |
+| 431 | [mctest-nli](src/tasksource/tasks.py#L1557) | Classification | [tasksource/mctest-nli](https://hf.co/datasets/tasksource/mctest-nli) |  |
+| 432 | [patent-phrase-similarity](src/tasksource/tasks.py#L1559) | Classification | [tasksource/patent-phrase-similarity](https://hf.co/datasets/tasksource/patent-phrase-similarity) |  |
+| 433 | [natural-language-satisfiability](src/tasksource/tasks.py#L1561) | Classification | [tasksource/natural-language-satisfiability](https://hf.co/datasets/tasksource/natural-language-satisfiability) |  |
+| 434 | [idioms-nli](src/tasksource/tasks.py#L1563) | Classification | [tasksource/idioms-nli](https://hf.co/datasets/tasksource/idioms-nli) |  |
+| 435 | [lifecycle-entailment](src/tasksource/tasks.py#L1565) | Classification | [tasksource/lifecycle-entailment](https://hf.co/datasets/tasksource/lifecycle-entailment) |  |
+| 436 | [safe-guard-prompt-injection](src/tasksource/tasks.py#L1572) | Classification | [xTRam1/safe-guard-prompt-injection](https://hf.co/datasets/xTRam1/safe-guard-prompt-injection) | ✓ |
+| 437 | [prompt-injections](src/tasksource/tasks.py#L1578) | Classification | [deepset/prompt-injections](https://hf.co/datasets/deepset/prompt-injections) | ✓ |
+| 438 | [prompt-injection-dataset](src/tasksource/tasks.py#L1584) | Classification | [S-Labs/prompt-injection-dataset](https://hf.co/datasets/S-Labs/prompt-injection-dataset) | ✓ |
+| 439 | [Prompt-injection-dataset/full](src/tasksource/tasks.py#L1590) | Classification | [neuralchemy/Prompt-injection-dataset](https://hf.co/datasets/neuralchemy/Prompt-injection-dataset) | ✓ |
+| 440 | [PromptShield](src/tasksource/tasks.py#L1596) | Classification | [hendzh/PromptShield](https://hf.co/datasets/hendzh/PromptShield) | ✓ |
+| 441 | [shell-safety-v2](src/tasksource/tasks.py#L1602) | Classification | [tomngdev/shell-safety-v2](https://hf.co/datasets/tomngdev/shell-safety-v2) | ✓ |
+| 442 | [agent_action_safety](src/tasksource/tasks.py#L1607) | Classification | json | ✓ |
+| 443 | [ShellRisk-Bench](src/tasksource/tasks.py#L1622) | Classification | [kontext-security/ShellRisk-Bench](https://hf.co/datasets/kontext-security/ShellRisk-Bench) | ✓ |
+| 444 | [wildguardmix-cleaned/prompt_harm](src/tasksource/tasks.py#L1647) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
+| 445 | [wildguardmix-cleaned/response_harm](src/tasksource/tasks.py#L1653) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
+| 446 | [wildguardmix-cleaned/response_refusal](src/tasksource/tasks.py#L1658) | Classification | [bogdanminko/wildguardmix-cleaned](https://hf.co/datasets/bogdanminko/wildguardmix-cleaned) | ✓ |
+| 447 | [BeaverTails](src/tasksource/tasks.py#L1674) | Classification | [PKU-Alignment/BeaverTails](https://hf.co/datasets/PKU-Alignment/BeaverTails) | ✓ |
+| 448 | [privacy-200k-Mistral-Large-3](src/tasksource/tasks.py#L1689) | Classification | [gabrielloiseau/privacy-200k-Mistral-Large-3](https://hf.co/datasets/gabrielloiseau/privacy-200k-Mistral-Large-3) | ✓ |
+| 449 | [toxic-chat/toxicchat0124/toxicity](src/tasksource/tasks.py#L1699) | Classification | [lmsys/toxic-chat](https://hf.co/datasets/lmsys/toxic-chat) | ✓ |
+| 450 | [toxic-chat/toxicchat0124/jailbreaking](src/tasksource/tasks.py#L1704) | Classification | [lmsys/toxic-chat](https://hf.co/datasets/lmsys/toxic-chat) | ✓ |
+| 451 | [clinc_oos/plus](src/tasksource/tasks.py#L1710) | Classification | [clinc/clinc_oos](https://hf.co/datasets/clinc/clinc_oos) |  |
+| 452 | [IntentGrasp/all](src/tasksource/tasks.py#L1728) | MultipleChoice | [yuweiyin/IntentGrasp](https://hf.co/datasets/yuweiyin/IntentGrasp) |  |
+| 453 | [few_rel/default](src/tasksource/tasks.py#L1766) | Classification | [tasksource/few_rel](https://hf.co/datasets/tasksource/few_rel) |  |
+| 454 | [docred](src/tasksource/tasks.py#L1812) | Classification | json |  |
+| 455 | [chemprot/chemprot_full_source](src/tasksource/tasks.py#L1840) | Classification | [bigbio/chemprot](https://hf.co/datasets/bigbio/chemprot) |  |
+| 456 | [PKU-SafeRLHF/helpfulness](src/tasksource/tasks.py#L1845) | MultipleChoice | [PKU-Alignment/PKU-SafeRLHF](https://hf.co/datasets/PKU-Alignment/PKU-SafeRLHF) | ✓ |
+| 457 | [PKU-SafeRLHF/safety](src/tasksource/tasks.py#L1850) | MultipleChoice | [PKU-Alignment/PKU-SafeRLHF](https://hf.co/datasets/PKU-Alignment/PKU-SafeRLHF) | ✓ |
+| 458 | [HelpSteer/helpfulness](src/tasksource/tasks.py#L1872) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
+| 459 | [HelpSteer/correctness](src/tasksource/tasks.py#L1873) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
+| 460 | [HelpSteer/coherence](src/tasksource/tasks.py#L1874) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
+| 461 | [HelpSteer/complexity](src/tasksource/tasks.py#L1875) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
+| 462 | [HelpSteer/verbosity](src/tasksource/tasks.py#L1876) | Classification | [nvidia/HelpSteer](https://hf.co/datasets/nvidia/HelpSteer) | ✓ |
+| 463 | [HelpSteer2/helpfulness](src/tasksource/tasks.py#L1878) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
+| 464 | [HelpSteer2/correctness](src/tasksource/tasks.py#L1879) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
+| 465 | [HelpSteer2/coherence](src/tasksource/tasks.py#L1880) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
+| 466 | [HelpSteer2/complexity](src/tasksource/tasks.py#L1881) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
+| 467 | [HelpSteer2/verbosity](src/tasksource/tasks.py#L1882) | Classification | [nvidia/HelpSteer2](https://hf.co/datasets/nvidia/HelpSteer2) | ✓ |
+| 468 | [HelpSteer3/preference](src/tasksource/tasks.py#L1887) | MultipleChoice | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
+| 469 | [HelpSteer3/principle](src/tasksource/tasks.py#L1892) | Classification | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) |  |
+| 470 | [HelpSteer3/edit_quality](src/tasksource/tasks.py#L1897) | MultipleChoice | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
+| 471 | [HelpSteer3/feedback](src/tasksource/tasks.py#L1920) | Classification | [nvidia/HelpSteer3](https://hf.co/datasets/nvidia/HelpSteer3) | ✓ |
+| 472 | [MSciNLI](src/tasksource/tasks.py#L1925) | Classification | [sadat2307/MSciNLI](https://hf.co/datasets/sadat2307/MSciNLI) |  |
+| 473 | [UltraFeedback-paired](src/tasksource/tasks.py#L1928) | MultipleChoice | [pushpdeep/UltraFeedback-paired](https://hf.co/datasets/pushpdeep/UltraFeedback-paired) | ✓ |
+| 474 | [prm800k_dpo/solution](src/tasksource/tasks.py#L1932) | MultipleChoice | [tasksource/prm800k_dpo](https://hf.co/datasets/tasksource/prm800k_dpo) | ✓ |
+| 475 | [prm800k_dpo/step](src/tasksource/tasks.py#L1935) | MultipleChoice | [tasksource/prm800k_dpo](https://hf.co/datasets/tasksource/prm800k_dpo) | ✓ |
+| 476 | [AES2-essay-scoring](src/tasksource/tasks.py#L1939) | Classification | [tasksource/AES2-essay-scoring](https://hf.co/datasets/tasksource/AES2-essay-scoring) | ✓ |
+| 477 | [argument-feedback](src/tasksource/tasks.py#L1943) | Classification | [tasksource/argument-feedback](https://hf.co/datasets/tasksource/argument-feedback) | ✓ |
+| 478 | [english-grading/cohesion](src/tasksource/tasks.py#L1950) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 479 | [english-grading/syntax](src/tasksource/tasks.py#L1951) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 480 | [english-grading/vocabulary](src/tasksource/tasks.py#L1952) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 481 | [english-grading/phraseology](src/tasksource/tasks.py#L1953) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 482 | [english-grading/grammar](src/tasksource/tasks.py#L1954) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 483 | [english-grading/conventions](src/tasksource/tasks.py#L1955) | Classification | [tasksource/english-grading](https://hf.co/datasets/tasksource/english-grading) | ✓ |
+| 484 | [wice](src/tasksource/tasks.py#L1957) | Classification | [tasksource/wice](https://hf.co/datasets/tasksource/wice) |  |
+| 485 | [hover](src/tasksource/tasks.py#L1960) | Classification | [Dzeniks/hover](https://hf.co/datasets/Dzeniks/hover) |  |
+| 486 | [hover-3way/nli](src/tasksource/tasks.py#L1964) | Classification | [Dzeniks/hover-3way](https://hf.co/datasets/Dzeniks/hover-3way) |  |
+| 487 | [tasksource_dpo_pairs](src/tasksource/tasks.py#L1967) | MultipleChoice | [tasksource/tasksource_dpo_pairs](https://hf.co/datasets/tasksource/tasksource_dpo_pairs) | ✓ |
+| 488 | [seahorse_summarization_evaluation](src/tasksource/tasks.py#L1970) | Classification | [tasksource/seahorse_summarization_evaluation](https://hf.co/datasets/tasksource/seahorse_summarization_evaluation) |  |
+| 489 | [missing-item-prediction/contrastive](src/tasksource/tasks.py#L1973) | Classification | [sileod/missing-item-prediction](https://hf.co/datasets/sileod/missing-item-prediction) |  |
+| 490 | [jigsaw_toxicity](src/tasksource/tasks.py#L1977) | Classification | [tasksource/jigsaw_toxicity](https://hf.co/datasets/tasksource/jigsaw_toxicity) |  |
+| 491 | [Pol_NLI](src/tasksource/tasks.py#L1980) | Classification | [mlburnham/Pol_NLI](https://hf.co/datasets/mlburnham/Pol_NLI) |  |
+| 492 | [synthetic-retrieval-NLI/position](src/tasksource/tasks.py#L1983) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
+| 493 | [synthetic-retrieval-NLI/binary](src/tasksource/tasks.py#L1983) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
+| 494 | [synthetic-retrieval-NLI/count](src/tasksource/tasks.py#L1983) | Classification | [tasksource/synthetic-retrieval-NLI](https://hf.co/datasets/tasksource/synthetic-retrieval-NLI) |  |
+| 495 | [github-issue-similarity](src/tasksource/tasks.py#L1992) | Classification | [WhereIsAI/github-issue-similarity](https://hf.co/datasets/WhereIsAI/github-issue-similarity) |  |