Add seqio tasks

evaluation/template_list.py

@@ -82,5 +82,25 @@
         "GPT-3-prompt-with-label",
         "polysemous",
         "similar-sense",
+    ],
+    ("hellaswag", None): [
+        "Predict ending with hint",
+        "Randomized prompts template",
+        "complete_first_then",
+        "if_begins_how_continues",
+    ],
+    ("super_glue", "copa"): [
+        "exercise",
+        "…What could happen next, C1 or C2?",
+        "i_am_hesitating",
+        "plausible_alternatives",
+        "C1 or C2? premise, so/because…",
+        "…As a result, C1 or C2?",
+        "best_option",
+        "…which may be caused by",
+        "more likely",
+        "cause_effect",
+        "…why? C1 or C2",
+        "choose",
     ]
 }

requirements.txt

@@ -1,9 +0,0 @@
-git+git://github.com/bigscience-workshop/[email protected]
-accelerate
-transformers
-datasets
-jinja2
-torch
-sentencepiece
-protobuf
-scikit-learn

setup.py

@@ -22,4 +22,28 @@
     packages=find_packages(),
     license="Apache Software License 2.0",
     long_description=readme,
+    install_requires=[
+        "promptsource @ git+git://github.com/bigscience-workshop/[email protected]",
+        "accelerate",
+        "transformers",
+        "torch",
+        "datasets",
+        "jinja2",
+        "datasets",
+        "sentencepiece",
+        "protobuf",
+        "scikit-learn"
+    ],
+    extras_require={
+        "seqio_tasks": [
+            "seqio",
+            "t5",
+            "tensorflow",
+        ]
+    },
+    package_data={
+        "": [
+            "seqio_tasks/datasets.csv",
+        ]
+    }
 )

t0/datasets.csv

@@ -0,0 +1,67 @@
+HF_name,subset,task_by_convention,do_train,do_eval,train_size
+crows_pairs,,bias_and_fairness,,BIAS_FAIRNESS,
+jigsaw_toxicity_pred,,bias_and_fairness,,BIAS_FAIRNESS,
+super_glue,axg,bias_and_fairness,,BIAS_FAIRNESS,
+wino_bias,type1_anti,bias_and_fairness,,BIAS_FAIRNESS,
+wino_bias,type2_anti,bias_and_fairness,,BIAS_FAIRNESS,
+wino_bias,type1_pro,bias_and_fairness,,BIAS_FAIRNESS,
+wino_bias,type2_pro,bias_and_fairness,,BIAS_FAIRNESS,
+super_glue,wsc.fixed,coreference,SGLUE,BASE,554
+winogrande,winogrande_xl,coreference,,BASE,40398
+super_glue,cb,NLI,,BASE,250
+super_glue,rte,NLI,,BASE,2490
+anli,,NLI,,BASE,162865
+glue,mrpc,paraphrase,BASE,,3668
+glue,qqp,paraphrase,BASE,,363846
+paws,labeled_final,paraphrase,BASE,,49401
+ai2_arc,ARC-Challenge,QA_closed_book,GPT_EVAL,,1119
+ai2_arc,ARC-Easy,QA_closed_book,GPT_EVAL,,2251
+kilt_tasks,hotpotqa,QA_closed_book,BASE,,88869
+trivia_qa,unfiltered,QA_closed_book,GPT_EVAL,,87622
+web_questions,,QA_closed_book,GPT_EVAL,,3778
+wiki_qa,,QA_closed_book,BASE,,20360
+adversarial_qa,dbidaf,QA_extractive,BASE,,10000
+adversarial_qa,dbert,QA_extractive,BASE,,10000
+adversarial_qa,droberta,QA_extractive,BASE,,10000
+duorc,SelfRC,QA_extractive,BASE,,60721
+duorc,ParaphraseRC,QA_extractive,BASE,,69524
+ropes,,QA_extractive,BASE,,10924
+squad_v2,,QA_extractive,GPT_EVAL,,130319
+super_glue,record,QA_extractive,SGLUE,,100730
+quoref,,QA_extractive,BASE,,19399
+cos_e,v1.11,QA_multiple_choice,BASE,,9741
+cosmos_qa,,QA_multiple_choice,BASE,,25262
+dream,,QA_multiple_choice,BASE,,6116
+openbookqa,main,QA_multiple_choice,GPT_EVAL,,4957
+qasc,,QA_multiple_choice,BASE,,8134
+quail,,QA_multiple_choice,BASE,,10246
+quarel,,QA_multiple_choice,BASE,,1941
+quartz,,QA_multiple_choice,BASE,,2696
+race,high,QA_multiple_choice,GPT_EVAL,,62445
+race,middle,QA_multiple_choice,GPT_EVAL,,25421
+sciq,,QA_multiple_choice,BASE,,11679
+social_i_qa,,QA_multiple_choice,BASE,,33410
+super_glue,boolq,QA_multiple_choice,SGLUE,,9427
+super_glue,copa,QA_multiple_choice,SGLUE,BASE,400
+super_glue,multirc,QA_multiple_choice,SGLUE,,27243
+wiki_hop,original,QA_multiple_choice,BASE,,43738
+wiqa,,QA_multiple_choice,BASE,,29808
+piqa,,QA_multiple_choice,GPT_EVAL,,16113
+amazon_polarity,,sentiment,BASE,,3600000
+app_reviews,,sentiment,BASE,,288065
+imdb,,sentiment,BASE,,25000
+rotten_tomatoes,,sentiment,BASE,,8530
+yelp_review_full,,sentiment,BASE,,650000
+story_cloze,2016,story_completion,,BASE,
+hellaswag,,story_completion,GPT_EVAL,BASE,39905
+common_gen,,structure_to_text,BASE,,67389
+wiki_bio,,structure_to_text,BASE,,582659
+cnn_dailymail,3.0.0,summarization,BASE,,287113
+gigaword,,summarization,BASE,,3803957
+multi_news,,summarization,BASE,,44972
+samsum,,summarization,BASE,,14732
+xsum,,summarization,BASE,,204045
+ag_news,,topic_classification,BASE,,120000
+dbpedia_14,,topic_classification,BASE,,560000
+trec,,topic_classification,BASE,,5452
+super_glue,wic,word_sense_disambiguation,SGLUE,BASE,5428

t0/seqio_tasks/__init__.py

@@ -0,0 +1,3 @@
+"""Tools for loading prompted tasks in seqio."""
+
+from t0.seqio_tasks import tasks, utils