diff --git a/prepare/cards/arc.py b/prepare/cards/arc.py index d556f949f7..25650a3b14 100644 --- a/prepare/cards/arc.py +++ b/prepare/cards/arc.py @@ -7,7 +7,7 @@ for subtask in subtasks: card = TaskCard( - loader=LoadHF(path="ai2_arc", name=subtask), + loader=LoadHF(path="allenai/ai2_arc", name=subtask), preprocess_steps=[ Set({"topic": "science"}), Rename(field_to_field={"answerKey": "label", "choices": "_choices"}), diff --git a/prepare/cards/billsum.py b/prepare/cards/billsum.py index d0c8055423..dbc0a06c5d 100644 --- a/prepare/cards/billsum.py +++ b/prepare/cards/billsum.py @@ -10,7 +10,7 @@ n_chars_to_filter_by_list = ["max", 6000, 10000] for n_chars_to_filter_by in n_chars_to_filter_by_list: card = TaskCard( - loader=LoadHF(path="billsum"), + loader=LoadHF(path="FiscalNote/billsum"), preprocess_steps=[ SplitRandomMix( {"train": "train[87.5%]", "validation": "train[12.5%]", "test": "test"} diff --git a/prepare/cards/clinc_oos.py b/prepare/cards/clinc_oos.py index e73568d312..68a32429bb 100644 --- a/prepare/cards/clinc_oos.py +++ b/prepare/cards/clinc_oos.py @@ -171,7 +171,7 @@ for subset in ["small", "imbalanced", "plus"]: card = TaskCard( - loader=LoadHF(path="clinc_oos", name=subset), + loader=LoadHF(path="clinc/clinc_oos", name=subset), preprocess_steps=[ Shuffle(page_size=sys.maxsize), Rename(field_to_field={"intent": "label"}), diff --git a/prepare/cards/cnn_dailymail.py b/prepare/cards/cnn_dailymail.py index ee7c37cc46..ff7dbd13dc 100644 --- a/prepare/cards/cnn_dailymail.py +++ b/prepare/cards/cnn_dailymail.py @@ -9,7 +9,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="cnn_dailymail", name="3.0.0"), + loader=LoadHF(path="abisee/cnn_dailymail", name="3.0.0"), preprocess_steps=[ Rename(field_to_field={"article": "document"}), Wrap(field="highlights", inside="list", to_field="summaries"), diff --git a/prepare/cards/copa.py b/prepare/cards/copa.py index 8ae00e9cf5..d6e0172ac5 100644 --- a/prepare/cards/copa.py +++ b/prepare/cards/copa.py @@ -10,7 +10,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="super_glue", name="copa"), + loader=LoadHF(path="aps/super_glue", name="copa"), preprocess_steps=[ "splitters.small_no_test", ListFieldValues(fields=["choice1", "choice2"], to_field="choices"), diff --git a/prepare/cards/dbpedia_14.py b/prepare/cards/dbpedia_14.py index 63cb2c6f53..3744e11f4e 100644 --- a/prepare/cards/dbpedia_14.py +++ b/prepare/cards/dbpedia_14.py @@ -34,7 +34,7 @@ mappers = {str(i): cls for i, cls in enumerate(classes)} card = TaskCard( - loader=LoadHF(path=f"{dataset_name}"), + loader=LoadHF(path="fancyzhx/dbpedia_14"), preprocess_steps=[ Shuffle(page_size=sys.maxsize), SplitRandomMix( diff --git a/prepare/cards/ethos.py b/prepare/cards/ethos.py index 3ad32db9f6..298054b44b 100644 --- a/prepare/cards/ethos.py +++ b/prepare/cards/ethos.py @@ -12,7 +12,7 @@ card = TaskCard( loader=LoadHF( - path="ethos", + path="iamollas/ethos", revision="refs/convert/parquet", data_dir="binary", splits=["train"], diff --git a/prepare/cards/go_emotions.py b/prepare/cards/go_emotions.py index 97cf5911fa..e5c0c061b9 100644 --- a/prepare/cards/go_emotions.py +++ b/prepare/cards/go_emotions.py @@ -11,13 +11,13 @@ dataset_name = "go_emotions" subset = "simplified" -ds_builder = load_dataset_builder(dataset_name, subset) +ds_builder = load_dataset_builder("google-research-datasets/go_emotions", subset) classes = ds_builder.info.features["labels"].feature.names mappers = {str(i): cls for i, cls in enumerate(classes)} card = TaskCard( - loader=LoadHF(path=dataset_name, name=subset), + loader=LoadHF(path="google-research-datasets/go_emotions", name=subset), preprocess_steps=[ MapInstanceValues(mappers={"labels": mappers}, process_every_value=True), Set( diff --git a/prepare/cards/hellaswag.py b/prepare/cards/hellaswag.py index 1162f40d5e..0a77bdab84 100644 --- a/prepare/cards/hellaswag.py +++ b/prepare/cards/hellaswag.py @@ -4,7 +4,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="hellaswag"), + loader=LoadHF(path="Rowan/hellaswag"), preprocess_steps=[ "splitters.large_no_test", Rename( diff --git a/prepare/cards/human_eval.py b/prepare/cards/human_eval.py index 1e8f7ca3ba..a5014e30e6 100644 --- a/prepare/cards/human_eval.py +++ b/prepare/cards/human_eval.py @@ -15,7 +15,7 @@ get_asserts = '[t for t in re.findall(r"assert.*?(?=\\n\\s*assert|$)", test.replace("candidate", entry_point), re.DOTALL)]' card = TaskCard( - loader=LoadHF(path="openai_humaneval", split="test"), + loader=LoadHF(path="openai/openai_humaneval", split="test"), preprocess_steps=[ ExecuteExpression( expression=get_asserts, imports_list=["re"], to_field="test_list" diff --git a/prepare/cards/ledgar.py b/prepare/cards/ledgar.py index 5a29807dba..9716799e40 100644 --- a/prepare/cards/ledgar.py +++ b/prepare/cards/ledgar.py @@ -10,7 +10,7 @@ dataset_name = "ledgar" -ds_builder = load_dataset_builder("lex_glue", dataset_name) +ds_builder = load_dataset_builder("coastalcph/lex_glue", dataset_name) classlabels = ds_builder.info.features["label"] mappers = {} @@ -18,7 +18,7 @@ mappers[str(i)] = classlabels.names[i] card = TaskCard( - loader=LoadHF(path="lex_glue", name=f"{dataset_name}"), + loader=LoadHF(path="coastalcph/lex_glue", name=f"{dataset_name}"), preprocess_steps=[ MapInstanceValues({"label": mappers}), Set( diff --git a/prepare/cards/mbpp.py b/prepare/cards/mbpp.py index 51da1d2924..0dc7382921 100644 --- a/prepare/cards/mbpp.py +++ b/prepare/cards/mbpp.py @@ -11,7 +11,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="mbpp", name="full", split="test"), + loader=LoadHF(path="google-research-datasets/mbpp", name="full", split="test"), preprocess_steps=[ JoinStr(field_to_field={"test_list": "test_list_str"}, separator=os.linesep), ], diff --git a/prepare/cards/mlsum.py b/prepare/cards/mlsum.py index 50e28ad039..75b551835e 100644 --- a/prepare/cards/mlsum.py +++ b/prepare/cards/mlsum.py @@ -12,7 +12,7 @@ for lang in langs: card = TaskCard( loader=LoadHF( - path="mlsum", + path="reciTAL/mlsum", revision="refs/convert/parquet", data_dir=lang, splits=["train", "test", "validation"], diff --git a/prepare/cards/openbookqa.py b/prepare/cards/openbookqa.py index 309c04b942..8b64245af3 100644 --- a/prepare/cards/openbookqa.py +++ b/prepare/cards/openbookqa.py @@ -4,7 +4,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="openbookqa"), + loader=LoadHF(path="allenai/openbookqa"), preprocess_steps=[ Rename( field_to_field={"choices/text": "choices_text", "choices/label": "labels"}, diff --git a/prepare/cards/piqa.py b/prepare/cards/piqa.py index 0bc2f352a5..68cea70fbf 100644 --- a/prepare/cards/piqa.py +++ b/prepare/cards/piqa.py @@ -4,7 +4,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="piqa", revision="refs/pr/9"), + loader=LoadHF(path="ybisk/piqa", revision="refs/pr/9"), preprocess_steps=[ ListFieldValues(fields=["sol1", "sol2"], to_field="choices"), Rename( diff --git a/prepare/cards/race.py b/prepare/cards/race.py index 06bc875b80..53fa8ef4c7 100644 --- a/prepare/cards/race.py +++ b/prepare/cards/race.py @@ -7,7 +7,7 @@ for subset in ["all", "high", "middle"]: card = TaskCard( - loader=LoadHF(path="race", name=subset), + loader=LoadHF(path="ehovy/race", name=subset), preprocess_steps=[ Set({"numbering": numbering}), IndexOf(search_in="numbering", index_of="answer", to_field="answer"), diff --git a/prepare/cards/sciq.py b/prepare/cards/sciq.py index e9d4c456d4..8fac884575 100644 --- a/prepare/cards/sciq.py +++ b/prepare/cards/sciq.py @@ -10,7 +10,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="sciq"), + loader=LoadHF(path="allenai/sciq"), preprocess_steps=[ ListFieldValues( fields=["distractor1", "distractor2", "distractor3", "correct_answer"], diff --git a/prepare/cards/squad.py b/prepare/cards/squad.py index 5d819ce6c4..f34072481c 100644 --- a/prepare/cards/squad.py +++ b/prepare/cards/squad.py @@ -4,7 +4,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="squad"), + loader=LoadHF(path="rajpurkar/squad"), preprocess_steps=[ "splitters.small_no_test", Copy(field="answers/text", to_field="answers"), diff --git a/prepare/cards/translation/wmt/en_de.py b/prepare/cards/translation/wmt/en_de.py index a091173d48..374df88d99 100644 --- a/prepare/cards/translation/wmt/en_de.py +++ b/prepare/cards/translation/wmt/en_de.py @@ -3,7 +3,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="wmt16", name="de-en", streaming=True), + loader=LoadHF(path="wmt/wmt16", name="de-en", streaming=True), preprocess_steps=[ Copy( field_to_field=[ diff --git a/prepare/cards/translation/wmt/en_fr.py b/prepare/cards/translation/wmt/en_fr.py index 2cf30196a9..b4789386cb 100644 --- a/prepare/cards/translation/wmt/en_fr.py +++ b/prepare/cards/translation/wmt/en_fr.py @@ -3,7 +3,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="wmt14", name="fr-en", streaming=True), + loader=LoadHF(path="wmt/wmt14", name="fr-en", streaming=True), preprocess_steps=[ Copy( field_to_field=[ diff --git a/prepare/cards/translation/wmt/en_ro.py b/prepare/cards/translation/wmt/en_ro.py index 7a07945c50..72e8c0265b 100644 --- a/prepare/cards/translation/wmt/en_ro.py +++ b/prepare/cards/translation/wmt/en_ro.py @@ -3,7 +3,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="wmt16", name="ro-en", streaming=True), + loader=LoadHF(path="wmt/wmt16", name="ro-en", streaming=True), preprocess_steps=[ Copy( field_to_field=[ diff --git a/prepare/cards/trec.py b/prepare/cards/trec.py index b74f054f85..72c2467ca5 100644 --- a/prepare/cards/trec.py +++ b/prepare/cards/trec.py @@ -124,7 +124,7 @@ card = TaskCard( loader=LoadHF( - path="trec", revision="refs/convert/parquet", splits=["train", "test"] + path="CogComp/trec", revision="refs/convert/parquet", splits=["train", "test"] ), preprocess_steps=[ Shuffle(page_size=sys.maxsize), diff --git a/prepare/cards/unfair_tos.py b/prepare/cards/unfair_tos.py index e79594708d..16305dc3c8 100644 --- a/prepare/cards/unfair_tos.py +++ b/prepare/cards/unfair_tos.py @@ -11,7 +11,7 @@ dataset_name = "unfair_tos" -ds_builder = load_dataset_builder("lex_glue", dataset_name) +ds_builder = load_dataset_builder("coastalcph/lex_glue", dataset_name) classlabels = ds_builder.info.features["labels"] mappers = {} @@ -19,7 +19,7 @@ mappers[str(i)] = classlabels.feature.names[i] card = TaskCard( - loader=LoadHF(path="lex_glue", name=f"{dataset_name}"), + loader=LoadHF(path="coastalcph/lex_glue", name=f"{dataset_name}"), preprocess_steps=[ MapInstanceValues(mappers={"labels": mappers}, process_every_value=True), Set( diff --git a/prepare/cards/wiki_bio.py b/prepare/cards/wiki_bio.py index a70a1e7b43..880121c64d 100644 --- a/prepare/cards/wiki_bio.py +++ b/prepare/cards/wiki_bio.py @@ -11,7 +11,7 @@ card = TaskCard( loader=LoadHF( - path="wiki_bio", + path="michaelauli/wiki_bio", revision="refs/convert/parquet", splits=["train", "validation", "test"], ), diff --git a/prepare/cards/winogrande.py b/prepare/cards/winogrande.py index ae5b8f9f65..7c605af2e1 100644 --- a/prepare/cards/winogrande.py +++ b/prepare/cards/winogrande.py @@ -6,7 +6,9 @@ for subtask in ["debiased", "l", "m", "s", "xl", "xs"]: card = TaskCard( loader=LoadHF( - path="winogrande", name=f"winogrande_{subtask}", revision="refs/pr/6" + path="allenai/winogrande", + name=f"winogrande_{subtask}", + revision="refs/pr/6", ), preprocess_steps=[ "splitters.small_no_test", diff --git a/prepare/cards/wsc.py b/prepare/cards/wsc.py index c0444636ee..25cc3077cf 100644 --- a/prepare/cards/wsc.py +++ b/prepare/cards/wsc.py @@ -10,7 +10,7 @@ from unitxt.test_utils.card import test_card card = TaskCard( - loader=LoadHF(path="super_glue", name="wsc"), + loader=LoadHF(path="aps/super_glue", name="wsc"), preprocess_steps=[ "splitters.small_no_test", MapInstanceValues(mappers={"label": {"0": "False", "1": "True"}}), diff --git a/prepare/cards/xnli.py b/prepare/cards/xnli.py index f3f0b1b892..3bf9354658 100644 --- a/prepare/cards/xnli.py +++ b/prepare/cards/xnli.py @@ -29,7 +29,7 @@ for lang in langs: card = TaskCard( - loader=LoadHF(path="xnli", name=lang), + loader=LoadHF(path="facebook/xnli", name=lang), preprocess_steps=[ "splitters.small_no_test", Rename(field_to_field={"premise": "text_a", "hypothesis": "text_b"}), diff --git a/prepare/cards/yahoo_answers_topics.py b/prepare/cards/yahoo_answers_topics.py index 91ea5923fc..1c13886b46 100644 --- a/prepare/cards/yahoo_answers_topics.py +++ b/prepare/cards/yahoo_answers_topics.py @@ -30,7 +30,7 @@ mappers = {str(i): cls for i, cls in enumerate(classes)} card = TaskCard( - loader=LoadHF(path=f"{dataset_name}"), + loader=LoadHF(path="community-datasets/yahoo_answers_topics"), preprocess_steps=[ Shuffle(page_size=sys.maxsize), SplitRandomMix( diff --git a/src/unitxt/catalog/cards/ai2_arc/arc_challenge.json b/src/unitxt/catalog/cards/ai2_arc/arc_challenge.json index d29f49187e..c6ebad6921 100644 --- a/src/unitxt/catalog/cards/ai2_arc/arc_challenge.json +++ b/src/unitxt/catalog/cards/ai2_arc/arc_challenge.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "ai2_arc", + "path": "allenai/ai2_arc", "name": "ARC-Challenge" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/ai2_arc/arc_easy.json b/src/unitxt/catalog/cards/ai2_arc/arc_easy.json index bc9155c8f7..aaa1cf9379 100644 --- a/src/unitxt/catalog/cards/ai2_arc/arc_easy.json +++ b/src/unitxt/catalog/cards/ai2_arc/arc_easy.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "ai2_arc", + "path": "allenai/ai2_arc", "name": "ARC-Easy" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/billsum.json b/src/unitxt/catalog/cards/billsum.json index 449260241a..94b1e5b215 100644 --- a/src/unitxt/catalog/cards/billsum.json +++ b/src/unitxt/catalog/cards/billsum.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "billsum" + "path": "FiscalNote/billsum" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/billsum_document_filtered_to_10000_chars.json b/src/unitxt/catalog/cards/billsum_document_filtered_to_10000_chars.json index 125b08082a..fef069d721 100644 --- a/src/unitxt/catalog/cards/billsum_document_filtered_to_10000_chars.json +++ b/src/unitxt/catalog/cards/billsum_document_filtered_to_10000_chars.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "billsum" + "path": "FiscalNote/billsum" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/billsum_document_filtered_to_6000_chars.json b/src/unitxt/catalog/cards/billsum_document_filtered_to_6000_chars.json index 6c90e5feef..f7071578ef 100644 --- a/src/unitxt/catalog/cards/billsum_document_filtered_to_6000_chars.json +++ b/src/unitxt/catalog/cards/billsum_document_filtered_to_6000_chars.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "billsum" + "path": "FiscalNote/billsum" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/clinc_oos/imbalanced.json b/src/unitxt/catalog/cards/clinc_oos/imbalanced.json index 3c3bb2ada3..4577a7380d 100644 --- a/src/unitxt/catalog/cards/clinc_oos/imbalanced.json +++ b/src/unitxt/catalog/cards/clinc_oos/imbalanced.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "clinc_oos", + "path": "clinc/clinc_oos", "name": "imbalanced" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/clinc_oos/plus.json b/src/unitxt/catalog/cards/clinc_oos/plus.json index a4bbf92d8b..9e9fbd1f80 100644 --- a/src/unitxt/catalog/cards/clinc_oos/plus.json +++ b/src/unitxt/catalog/cards/clinc_oos/plus.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "clinc_oos", + "path": "clinc/clinc_oos", "name": "plus" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/clinc_oos/small.json b/src/unitxt/catalog/cards/clinc_oos/small.json index 1678dc228f..f07bbea323 100644 --- a/src/unitxt/catalog/cards/clinc_oos/small.json +++ b/src/unitxt/catalog/cards/clinc_oos/small.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "clinc_oos", + "path": "clinc/clinc_oos", "name": "small" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/cnn_dailymail.json b/src/unitxt/catalog/cards/cnn_dailymail.json index 8c0d910f75..1bd8cfcc0e 100644 --- a/src/unitxt/catalog/cards/cnn_dailymail.json +++ b/src/unitxt/catalog/cards/cnn_dailymail.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "cnn_dailymail", + "path": "abisee/cnn_dailymail", "name": "3.0.0" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/copa.json b/src/unitxt/catalog/cards/copa.json index 1f21ef193f..859588b27a 100644 --- a/src/unitxt/catalog/cards/copa.json +++ b/src/unitxt/catalog/cards/copa.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "super_glue", + "path": "aps/super_glue", "name": "copa" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/dbpedia_14.json b/src/unitxt/catalog/cards/dbpedia_14.json index 0685ebd639..66cb2daf20 100644 --- a/src/unitxt/catalog/cards/dbpedia_14.json +++ b/src/unitxt/catalog/cards/dbpedia_14.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "dbpedia_14" + "path": "fancyzhx/dbpedia_14" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/ethos_binary.json b/src/unitxt/catalog/cards/ethos_binary.json index ed8529cc9f..dac6b91070 100644 --- a/src/unitxt/catalog/cards/ethos_binary.json +++ b/src/unitxt/catalog/cards/ethos_binary.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "ethos", + "path": "iamollas/ethos", "revision": "refs/convert/parquet", "data_dir": "binary", "splits": [ diff --git a/src/unitxt/catalog/cards/go_emotions/simplified.json b/src/unitxt/catalog/cards/go_emotions/simplified.json index 4537b9a2b6..01df590897 100644 --- a/src/unitxt/catalog/cards/go_emotions/simplified.json +++ b/src/unitxt/catalog/cards/go_emotions/simplified.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "go_emotions", + "path": "google-research-datasets/go_emotions", "name": "simplified" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/hellaswag.json b/src/unitxt/catalog/cards/hellaswag.json index 3bfdca901f..f11babf73a 100644 --- a/src/unitxt/catalog/cards/hellaswag.json +++ b/src/unitxt/catalog/cards/hellaswag.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "hellaswag" + "path": "Rowan/hellaswag" }, "preprocess_steps": [ "splitters.large_no_test", diff --git a/src/unitxt/catalog/cards/human_eval.json b/src/unitxt/catalog/cards/human_eval.json index 585590207e..8616b4799e 100644 --- a/src/unitxt/catalog/cards/human_eval.json +++ b/src/unitxt/catalog/cards/human_eval.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "openai_humaneval", + "path": "openai/openai_humaneval", "split": "test" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/ledgar.json b/src/unitxt/catalog/cards/ledgar.json index f1c2521c55..01cbb5cb8b 100644 --- a/src/unitxt/catalog/cards/ledgar.json +++ b/src/unitxt/catalog/cards/ledgar.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "lex_glue", + "path": "coastalcph/lex_glue", "name": "ledgar" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/mbpp.json b/src/unitxt/catalog/cards/mbpp.json index 833ef1ed09..58832f5de9 100644 --- a/src/unitxt/catalog/cards/mbpp.json +++ b/src/unitxt/catalog/cards/mbpp.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mbpp", + "path": "google-research-datasets/mbpp", "name": "full", "split": "test" }, diff --git a/src/unitxt/catalog/cards/mlsum/de.json b/src/unitxt/catalog/cards/mlsum/de.json index d18125436c..9d92814669 100644 --- a/src/unitxt/catalog/cards/mlsum/de.json +++ b/src/unitxt/catalog/cards/mlsum/de.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mlsum", + "path": "reciTAL/mlsum", "revision": "refs/convert/parquet", "data_dir": "de", "splits": [ diff --git a/src/unitxt/catalog/cards/mlsum/es.json b/src/unitxt/catalog/cards/mlsum/es.json index c1bcb82074..bb0b138587 100644 --- a/src/unitxt/catalog/cards/mlsum/es.json +++ b/src/unitxt/catalog/cards/mlsum/es.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mlsum", + "path": "reciTAL/mlsum", "revision": "refs/convert/parquet", "data_dir": "es", "splits": [ diff --git a/src/unitxt/catalog/cards/mlsum/fr.json b/src/unitxt/catalog/cards/mlsum/fr.json index 4809d993ba..3e9aa007f6 100644 --- a/src/unitxt/catalog/cards/mlsum/fr.json +++ b/src/unitxt/catalog/cards/mlsum/fr.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mlsum", + "path": "reciTAL/mlsum", "revision": "refs/convert/parquet", "data_dir": "fr", "splits": [ diff --git a/src/unitxt/catalog/cards/mlsum/ru.json b/src/unitxt/catalog/cards/mlsum/ru.json index 54f9aefe1e..30ea7d0f90 100644 --- a/src/unitxt/catalog/cards/mlsum/ru.json +++ b/src/unitxt/catalog/cards/mlsum/ru.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mlsum", + "path": "reciTAL/mlsum", "revision": "refs/convert/parquet", "data_dir": "ru", "splits": [ diff --git a/src/unitxt/catalog/cards/mlsum/tu.json b/src/unitxt/catalog/cards/mlsum/tu.json index 1a1e70e94e..807645e520 100644 --- a/src/unitxt/catalog/cards/mlsum/tu.json +++ b/src/unitxt/catalog/cards/mlsum/tu.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "mlsum", + "path": "reciTAL/mlsum", "revision": "refs/convert/parquet", "data_dir": "tu", "splits": [ diff --git a/src/unitxt/catalog/cards/openbook_qa.json b/src/unitxt/catalog/cards/openbook_qa.json index 6333f759ee..45e6776285 100644 --- a/src/unitxt/catalog/cards/openbook_qa.json +++ b/src/unitxt/catalog/cards/openbook_qa.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "openbookqa" + "path": "allenai/openbookqa" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/piqa.json b/src/unitxt/catalog/cards/piqa.json index 86fafcc39c..b923cb7623 100644 --- a/src/unitxt/catalog/cards/piqa.json +++ b/src/unitxt/catalog/cards/piqa.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "piqa", + "path": "ybisk/piqa", "revision": "refs/pr/9" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/race_all.json b/src/unitxt/catalog/cards/race_all.json index a986719f8e..36fcd32839 100644 --- a/src/unitxt/catalog/cards/race_all.json +++ b/src/unitxt/catalog/cards/race_all.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "race", + "path": "ehovy/race", "name": "all" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/race_high.json b/src/unitxt/catalog/cards/race_high.json index 0174a23e47..3cff1d9a55 100644 --- a/src/unitxt/catalog/cards/race_high.json +++ b/src/unitxt/catalog/cards/race_high.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "race", + "path": "ehovy/race", "name": "high" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/race_middle.json b/src/unitxt/catalog/cards/race_middle.json index 5d28b86493..f6c9c1e9e1 100644 --- a/src/unitxt/catalog/cards/race_middle.json +++ b/src/unitxt/catalog/cards/race_middle.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "race", + "path": "ehovy/race", "name": "middle" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/sciq.json b/src/unitxt/catalog/cards/sciq.json index ea52c97755..6aa86c81a0 100644 --- a/src/unitxt/catalog/cards/sciq.json +++ b/src/unitxt/catalog/cards/sciq.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "sciq" + "path": "allenai/sciq" }, "preprocess_steps": [ { diff --git a/src/unitxt/catalog/cards/squad.json b/src/unitxt/catalog/cards/squad.json index c2dc05f195..4e67e79ea5 100644 --- a/src/unitxt/catalog/cards/squad.json +++ b/src/unitxt/catalog/cards/squad.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "squad" + "path": "rajpurkar/squad" }, "preprocess_steps": [ "splitters.small_no_test", diff --git a/src/unitxt/catalog/cards/trec.json b/src/unitxt/catalog/cards/trec.json index e4aa7692ca..079481bf67 100644 --- a/src/unitxt/catalog/cards/trec.json +++ b/src/unitxt/catalog/cards/trec.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "trec", + "path": "CogComp/trec", "revision": "refs/convert/parquet", "splits": [ "train", diff --git a/src/unitxt/catalog/cards/unfair_tos.json b/src/unitxt/catalog/cards/unfair_tos.json index ce904f5e0f..d00d7a8c28 100644 --- a/src/unitxt/catalog/cards/unfair_tos.json +++ b/src/unitxt/catalog/cards/unfair_tos.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "lex_glue", + "path": "coastalcph/lex_glue", "name": "unfair_tos" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/wiki_bio.json b/src/unitxt/catalog/cards/wiki_bio.json index b6eb2931d0..8735051f20 100644 --- a/src/unitxt/catalog/cards/wiki_bio.json +++ b/src/unitxt/catalog/cards/wiki_bio.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "wiki_bio", + "path": "michaelauli/wiki_bio", "revision": "refs/convert/parquet", "splits": [ "train", diff --git a/src/unitxt/catalog/cards/winogrande/debiased.json b/src/unitxt/catalog/cards/winogrande/debiased.json index 0172b4f1bf..4a8fd0163c 100644 --- a/src/unitxt/catalog/cards/winogrande/debiased.json +++ b/src/unitxt/catalog/cards/winogrande/debiased.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_debiased", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/winogrande/l.json b/src/unitxt/catalog/cards/winogrande/l.json index 0e07322e08..c1c6fdadee 100644 --- a/src/unitxt/catalog/cards/winogrande/l.json +++ b/src/unitxt/catalog/cards/winogrande/l.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_l", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/winogrande/m.json b/src/unitxt/catalog/cards/winogrande/m.json index d6fcfaf4fa..df4766d18b 100644 --- a/src/unitxt/catalog/cards/winogrande/m.json +++ b/src/unitxt/catalog/cards/winogrande/m.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_m", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/winogrande/s.json b/src/unitxt/catalog/cards/winogrande/s.json index a485cfce6d..385e2d3131 100644 --- a/src/unitxt/catalog/cards/winogrande/s.json +++ b/src/unitxt/catalog/cards/winogrande/s.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_s", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/winogrande/xl.json b/src/unitxt/catalog/cards/winogrande/xl.json index 63fb15ca68..89fb79050c 100644 --- a/src/unitxt/catalog/cards/winogrande/xl.json +++ b/src/unitxt/catalog/cards/winogrande/xl.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_xl", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/winogrande/xs.json b/src/unitxt/catalog/cards/winogrande/xs.json index e3304457e9..277f82e148 100644 --- a/src/unitxt/catalog/cards/winogrande/xs.json +++ b/src/unitxt/catalog/cards/winogrande/xs.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "winogrande", + "path": "allenai/winogrande", "name": "winogrande_xs", "revision": "refs/pr/6" }, diff --git a/src/unitxt/catalog/cards/wmt/en_de.json b/src/unitxt/catalog/cards/wmt/en_de.json index dda4e0bede..71b0d4ae0d 100644 --- a/src/unitxt/catalog/cards/wmt/en_de.json +++ b/src/unitxt/catalog/cards/wmt/en_de.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "wmt16", + "path": "wmt/wmt16", "name": "de-en", "streaming": true }, diff --git a/src/unitxt/catalog/cards/wmt/en_fr.json b/src/unitxt/catalog/cards/wmt/en_fr.json index d6ad7e186c..b32545b5fb 100644 --- a/src/unitxt/catalog/cards/wmt/en_fr.json +++ b/src/unitxt/catalog/cards/wmt/en_fr.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "wmt14", + "path": "wmt/wmt14", "name": "fr-en", "streaming": true }, diff --git a/src/unitxt/catalog/cards/wmt/en_ro.json b/src/unitxt/catalog/cards/wmt/en_ro.json index 4e2e513857..b4f006b017 100644 --- a/src/unitxt/catalog/cards/wmt/en_ro.json +++ b/src/unitxt/catalog/cards/wmt/en_ro.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "wmt16", + "path": "wmt/wmt16", "name": "ro-en", "streaming": true }, diff --git a/src/unitxt/catalog/cards/wsc.json b/src/unitxt/catalog/cards/wsc.json index 0238e6befb..7fa4d4f190 100644 --- a/src/unitxt/catalog/cards/wsc.json +++ b/src/unitxt/catalog/cards/wsc.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "super_glue", + "path": "aps/super_glue", "name": "wsc" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/ar.json b/src/unitxt/catalog/cards/xnli/ar.json index 5841ac36cd..c04ae285dc 100644 --- a/src/unitxt/catalog/cards/xnli/ar.json +++ b/src/unitxt/catalog/cards/xnli/ar.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "ar" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/bg.json b/src/unitxt/catalog/cards/xnli/bg.json index b7b88e79fe..f75987877a 100644 --- a/src/unitxt/catalog/cards/xnli/bg.json +++ b/src/unitxt/catalog/cards/xnli/bg.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "bg" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/de.json b/src/unitxt/catalog/cards/xnli/de.json index 248e0bd978..38853b926f 100644 --- a/src/unitxt/catalog/cards/xnli/de.json +++ b/src/unitxt/catalog/cards/xnli/de.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "de" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/el.json b/src/unitxt/catalog/cards/xnli/el.json index 1de81410e4..151cdc325d 100644 --- a/src/unitxt/catalog/cards/xnli/el.json +++ b/src/unitxt/catalog/cards/xnli/el.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "el" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/en.json b/src/unitxt/catalog/cards/xnli/en.json index 603b0d47a4..e4b3a5d135 100644 --- a/src/unitxt/catalog/cards/xnli/en.json +++ b/src/unitxt/catalog/cards/xnli/en.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "en" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/es.json b/src/unitxt/catalog/cards/xnli/es.json index bd712dfa71..098b8dd2c7 100644 --- a/src/unitxt/catalog/cards/xnli/es.json +++ b/src/unitxt/catalog/cards/xnli/es.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "es" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/fr.json b/src/unitxt/catalog/cards/xnli/fr.json index 70ecc12d64..907eb4ba14 100644 --- a/src/unitxt/catalog/cards/xnli/fr.json +++ b/src/unitxt/catalog/cards/xnli/fr.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "fr" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/hi.json b/src/unitxt/catalog/cards/xnli/hi.json index ff555026f6..cc15238be7 100644 --- a/src/unitxt/catalog/cards/xnli/hi.json +++ b/src/unitxt/catalog/cards/xnli/hi.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "hi" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/ru.json b/src/unitxt/catalog/cards/xnli/ru.json index 06ee99e931..36f9432043 100644 --- a/src/unitxt/catalog/cards/xnli/ru.json +++ b/src/unitxt/catalog/cards/xnli/ru.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "ru" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/sw.json b/src/unitxt/catalog/cards/xnli/sw.json index 0dc8be4511..245f063d3d 100644 --- a/src/unitxt/catalog/cards/xnli/sw.json +++ b/src/unitxt/catalog/cards/xnli/sw.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "sw" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/th.json b/src/unitxt/catalog/cards/xnli/th.json index c754554f2e..95b2b9f2c7 100644 --- a/src/unitxt/catalog/cards/xnli/th.json +++ b/src/unitxt/catalog/cards/xnli/th.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "th" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/tr.json b/src/unitxt/catalog/cards/xnli/tr.json index 9c0163f537..4124bbcb43 100644 --- a/src/unitxt/catalog/cards/xnli/tr.json +++ b/src/unitxt/catalog/cards/xnli/tr.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "tr" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/ur.json b/src/unitxt/catalog/cards/xnli/ur.json index 6f8a14917b..dd13f31b7f 100644 --- a/src/unitxt/catalog/cards/xnli/ur.json +++ b/src/unitxt/catalog/cards/xnli/ur.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "ur" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/vi.json b/src/unitxt/catalog/cards/xnli/vi.json index 2fca66ecf0..4d5dc5a0c1 100644 --- a/src/unitxt/catalog/cards/xnli/vi.json +++ b/src/unitxt/catalog/cards/xnli/vi.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "vi" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/xnli/zh.json b/src/unitxt/catalog/cards/xnli/zh.json index 0ddd997941..23846daf21 100644 --- a/src/unitxt/catalog/cards/xnli/zh.json +++ b/src/unitxt/catalog/cards/xnli/zh.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "xnli", + "path": "facebook/xnli", "name": "zh" }, "preprocess_steps": [ diff --git a/src/unitxt/catalog/cards/yahoo_answers_topics.json b/src/unitxt/catalog/cards/yahoo_answers_topics.json index e95d189552..8d6ee58121 100644 --- a/src/unitxt/catalog/cards/yahoo_answers_topics.json +++ b/src/unitxt/catalog/cards/yahoo_answers_topics.json @@ -2,7 +2,7 @@ "__type__": "task_card", "loader": { "__type__": "load_hf", - "path": "yahoo_answers_topics" + "path": "community-datasets/yahoo_answers_topics" }, "preprocess_steps": [ { diff --git a/src/unitxt/metrics.py b/src/unitxt/metrics.py index 64ac09be19..9942c1be6d 100644 --- a/src/unitxt/metrics.py +++ b/src/unitxt/metrics.py @@ -3184,12 +3184,21 @@ class F1(GlobalMetric): prediction_type = str single_reference_per_prediction = True - _requirements_list: List[str] = ["scikit-learn<=1.5.2"] + _requirements_list: List[str] = ["scikit-learn"] + + _sklearn_metric_fn = None def prepare(self): super().prepare() - self._metric = hf_evaluate_load(self.metric) + from sklearn.metrics import f1_score, precision_score, recall_score + + metric_fn_map = { + "f1": f1_score, + "precision": precision_score, + "recall": recall_score, + } + self._sklearn_metric_fn = metric_fn_map[self.metric] def get_str_id(self, str): if str not in self.str_to_id: @@ -3209,26 +3218,25 @@ def compute( formatted_references = [ self.get_str_id(reference[0]) for reference in references ] - self.str_to_id.keys() formatted_predictions = [ self.get_str_id(prediction) for prediction in predictions ] labels = list(set(formatted_references)) - result = self._metric.compute( - predictions=formatted_predictions, - references=formatted_references, + score = self._sklearn_metric_fn( + y_true=formatted_references, + y_pred=formatted_predictions, labels=labels, average=self.average, ) - if isinstance(result[self.metric], numpy.ndarray): - final_result = {self.main_score: nan_mean(result[self.metric])} + if isinstance(score, numpy.ndarray): + final_result = {self.main_score: nan_mean(score)} for i, label in enumerate(labels): - final_result[f"{self.metric}_" + self.id_to_str[label]] = result[ - self.metric - ][i] + final_result[f"{self.metric}_" + self.id_to_str[label]] = float( + score[i] + ) else: - final_result = {self.main_score: result[self.metric]} + final_result = {self.main_score: float(score)} return final_result @@ -3477,10 +3485,19 @@ class F1MultiLabel(GlobalMetric, PackageRequirementsMixin): single_reference_per_prediction = True _requirements_list = ["scikit-learn"] + _sklearn_metric_fn = None + def prepare(self): super().prepare() - self._metric = hf_evaluate_load(self.metric, "multilabel") + from sklearn.metrics import f1_score, precision_score, recall_score + + metric_fn_map = { + "f1": f1_score, + "precision": precision_score, + "recall": recall_score, + } + self._sklearn_metric_fn = metric_fn_map[self.metric] def add_str_to_id(self, str): if str not in self.str_to_id: @@ -3530,21 +3547,25 @@ def compute( else: labels_param = None - result = self._metric.compute( - predictions=formatted_predictions, - references=formatted_references, - average=self.average, - labels=labels_param, - ) - if isinstance(result[self.metric], numpy.ndarray): - assert ( - len(result[self.metric]) == len(labels) - ), f"F1 result ({result[self.metric]}) has more entries than labels ({labels})" - final_result = {self.main_score: nan_mean(result[self.metric])} + kwargs = { + "y_pred": formatted_predictions, + "y_true": formatted_references, + "average": self.average, + } + if labels_param is not None: + kwargs["labels"] = labels_param + + score = self._sklearn_metric_fn(**kwargs) + + if isinstance(score, numpy.ndarray): + assert len(score) == len( + labels + ), f"F1 result ({score}) has more entries than labels ({labels})" + final_result = {self.main_score: nan_mean(score)} for i, label in enumerate(labels): - final_result[self.metric + "_" + label] = result[self.metric][i] + final_result[self.metric + "_" + label] = float(score[i]) else: - final_result = {self.main_score: result[self.metric]} + final_result = {self.main_score: float(score)} return final_result diff --git a/src/unitxt/text2sql_utils.py b/src/unitxt/text2sql_utils.py index 6c02d96a90..9c688885b1 100644 --- a/src/unitxt/text2sql_utils.py +++ b/src/unitxt/text2sql_utils.py @@ -855,11 +855,9 @@ def rows_to_multisets(df): return [row_to_multiset(row) for row in df.values] def sort_df(df): - sorted_df = df.copy() + sorted_df = df.astype(str).copy() for i in range(len(sorted_df.columns)): - sorted_df.iloc[:, i] = ( - sorted_df.iloc[:, i].astype(str).sort_values(ignore_index=True) - ) + sorted_df.iloc[:, i] = sorted_df.iloc[:, i].sort_values(ignore_index=True) return sorted_df if df1.empty or df2.empty or len(df1) != len(df2): diff --git a/tests/library/test_api.py b/tests/library/test_api.py index 1af86397f8..ef53f42ab3 100644 --- a/tests/library/test_api.py +++ b/tests/library/test_api.py @@ -520,7 +520,7 @@ def test_produce_with_recipe_with_list_of_instances(self): def test_load_dataset_from_dict(self): card = TaskCard( - loader=LoadHF(path="glue", name="wnli"), + loader=LoadHF(path="nyu-mll/glue", name="wnli"), task=Task( input_fields=["sentence1", "sentence2"], reference_fields=["label"], diff --git a/tests/library/test_card.py b/tests/library/test_card.py index 9dcc5f1346..95f0446e21 100644 --- a/tests/library/test_card.py +++ b/tests/library/test_card.py @@ -12,7 +12,7 @@ from tests.utils import UnitxtTestCase card = TaskCard( - loader=LoadHF(path="glue", name="wnli"), + loader=LoadHF(path="nyu-mll/glue", name="wnli"), preprocess_steps=[ SplitRandomMix( {"train": "train[95%]", "validation": "train[5%]", "test": "validation"} diff --git a/tests/library/test_loaders.py b/tests/library/test_loaders.py index bbf8fc114b..4540924af6 100644 --- a/tests/library/test_loaders.py +++ b/tests/library/test_loaders.py @@ -216,7 +216,7 @@ def test_load_from_ibm_cos(self): ) def test_load_from_HF(self): - loader = LoadHF(path="sst2", loader_limit=10, split="train") + loader = LoadHF(path="stanfordnlp/sst2", loader_limit=10, split="train") ms = loader() instance = next(iter(ms["train"])) self.assertEqual( @@ -272,7 +272,7 @@ def test_load_from_HF_multiple_innvocation_with_filter(self): self.assertEqual(instance["language"], "eng") def test_load_from_HF_split(self): - loader = LoadHF(path="sst2", split="train") + loader = LoadHF(path="stanfordnlp/sst2", split="train") ms = loader() instance = next(iter(ms["train"])) self.assertEqual(