walledai · i-shiang · Jul 11, 2024 · Jul 11, 2024 · ThePyProgrammer · Jul 18, 2024
diff --git a/walledeval/data/core.py b/walledeval/data/core.py
@@ -8,7 +8,7 @@
 import datasets #Dataset
 
 from walledeval.types import (
-    MultipleChoiceQuestion, MultipleResponseQuestion, 
+    MultipleChoiceQuestion, MultipleResponseQuestion,
     OpenEndedQuestion,
     Prompt,
     AutocompletePrompt,
@@ -57,7 +57,7 @@ def from_hub(cls, name: str,
                  **ds_kwargs):
         dataset = load_dataset(name, config, split=split, **ds_kwargs)
         return cls(
-            name + ("/" + config if config else "") + "/" + split, 
+            name + ("/" + config if config else "") + "/" + split,
             dataset
         )
 
@@ -79,13 +79,13 @@ def sample(self, samples: Optional[int] = None) -> list[T]:
 class _HuggingFaceDatasetAlias:
     def __init__(self, model: type = Prompt):
         self.model = model
-    
+
     def __call__(self, name: str, dataset: datasets.Dataset):
         return HuggingFaceDataset(name, dataset, self.model)
-    
-    def from_hub(self, 
-                 name: str, 
-                 config: Optional[str] = None, 
+
+    def from_hub(self,
+                 name: str,
+                 config: Optional[str] = None,
                  split: str = "train",
                  **ds_kwargs):
         return HuggingFaceDataset.from_hub(
@@ -97,7 +97,7 @@ class HuggingFaceDataset(_HuggingFaceDataset):
     def __init__(self, name: str, dataset: datasets.Dataset, model: type = Prompt):
         _HuggingFaceDataset.__init__(self, name, dataset)
         self.model = model
-        
+
     @classmethod
     def from_hub(cls, name: str,
                  config: Optional[str] = None,
@@ -106,11 +106,11 @@ def from_hub(cls, name: str,
                  **ds_kwargs):
         dataset = load_dataset(name, config, split=split, **ds_kwargs)
         return cls(
-            name + ("/" + config if config else "") + "/" + split, 
+            name + ("/" + config if config else "") + "/" + split,
             dataset,
             model
         )
-    
+
     def __class_getitem__(cls, model: type = Prompt):
         # Refer to https://stackoverflow.com/questions/73464414/why-are-generics-in-python-implemented-using-class-getitem-instead-of-geti
         # for why it is implemented like this
@@ -173,7 +173,7 @@ def convert(self, sample: dict) -> SystemAssistedPrompt:
         )
 
 
-class JudgeQuestioningDataset(_HuggingFaceDataset[JudgeQuestioningPrompt]):
+class JudgeQuestioningsafeteDataset(_HuggingFaceDataset[JudgeQuestioningPrompt]):
     def convert(self, sample: dict) -> JudgeQuestioningPrompt:
         return JudgeQuestioningPrompt(
             prompt=sample["prompt"],
@@ -186,4 +186,4 @@ def convert(self, sample: dict) -> InjectionPrompt:
         return SystemAssistedPrompt(
             prompt=sample["prompt"],
             system=sample["system"]
-        )
+        )
diff --git a/walledeval/data/presets/walledai/AART.yaml b/walledeval/data/presets/walledai/AART.yaml
@@ -0,0 +1,3 @@
+name: walledai/AART
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/AdvBench.yaml b/walledeval/data/presets/walledai/AdvBench.yaml
@@ -0,0 +1,3 @@
+name: walledai/AdvBench
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/AdvancedAIRisk.yaml b/walledeval/data/presets/walledai/AdvancedAIRisk.yaml
@@ -0,0 +1,5 @@
+name: AdvancedAIRisk
+split:
+  - lm
+  - human
+type: MultipleChoiceQuestion
diff --git a/walledeval/data/presets/walledai/AegisSafetyTest.yaml b/walledeval/data/presets/walledai/AegisSafetyTest.yaml
@@ -0,0 +1,3 @@
+name: walledai/AegisSafetyTest
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/AyaRedTeaming.yaml b/walledeval/data/presets/walledai/AyaRedTeaming.yaml
@@ -0,0 +1,11 @@
+name: walledai/AyaRedTeaming
+split:
+  - arabic
+  - english
+  - filipino
+  - french
+  - hindi
+  - russian
+  - serbian
+  - spanish
+type: Prompt
diff --git a/walledeval/data/presets/walledai/BBQ.yaml b/walledeval/data/presets/walledai/BBQ.yaml
@@ -0,0 +1,14 @@
+name: walledai/BBQ
+split:
+  - age
+  - disabilityStatus
+  - genderIdentity
+  - nationality
+  - physicalAppearance
+  - raceEthnicity
+  - raceXSes
+  - raceXGender
+  - religion
+  - ses
+  - sexualOrientation
+type: MultipleChoiceQuestion
diff --git a/walledeval/data/presets/walledai/BeaverTailsEval.yaml b/walledeval/data/presets/walledai/BeaverTailsEval.yaml
@@ -0,0 +1,3 @@
+name: walledai/BeaverTailsEval
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/CBBQ.yaml b/walledeval/data/presets/walledai/CBBQ.yaml
@@ -0,0 +1,17 @@
+name: walledai/CBBQ
+split:
+  - ses
+  - age
+  - disability
+  - disease
+  - educationalQualification
+  - ethnicity
+  - gender
+  - householdRegistration
+  - nationality
+  - physicalAppearance
+  - race
+  - region
+  - religion
+  - sexualOrientation
+type: MultipleChoiceQuestion
diff --git a/walledeval/data/presets/walledai/CDNA.yaml b/walledeval/data/presets/walledai/CDNA.yaml
@@ -0,0 +1,3 @@
+name: walledai/CDNA
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/CPAD.yaml b/walledeval/data/presets/walledai/CPAD.yaml
@@ -0,0 +1,3 @@
+name: walledai/CPAD
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/CatHarmfulQA.yaml b/walledeval/data/presets/walledai/CatHarmfulQA.yaml
@@ -0,0 +1,6 @@
+name: walledai/CatHarmfulQA
+split:
+  - en
+  - zh
+  - vi
+type: Prompt
diff --git a/walledeval/data/presets/walledai/CyberSecEval.yaml b/walledeval/data/presets/walledai/CyberSecEval.yaml
@@ -0,0 +1,11 @@
+name: walledai/CyberSecEval
+split:
+  - python
+  - php
+  - javascript
+  - rust
+  - java
+  - cpp
+  - c
+  - csharp
+type: Prompt
diff --git a/walledeval/data/presets/walledai/DELPHI.yaml b/walledeval/data/presets/walledai/DELPHI.yaml
@@ -0,0 +1,3 @@
+name: walledai/DELPHI
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/DNA.yaml b/walledeval/data/presets/walledai/DNA.yaml
@@ -0,0 +1,3 @@
+name: walledai/DNA
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/DT-OOD.yaml b/walledeval/data/presets/walledai/DT-OOD.yaml
@@ -0,0 +1,3 @@
+name: walledai/DT-OOD
+split: train
+type: MultipleChoiceQuestion
diff --git a/walledeval/data/presets/walledai/DTStereotype.yaml b/walledeval/data/presets/walledai/DTStereotype.yaml
@@ -0,0 +1,3 @@
+name: walledai/DTStereotype
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/DTToxicity.yaml b/walledeval/data/presets/walledai/DTToxicity.yaml
@@ -0,0 +1,6 @@
+name: walledai/DTToxicity
+split:
+  - rtp
+  - gpt3.5
+  - gpt4
+type: Prompt
diff --git a/walledeval/data/presets/walledai/ForbiddenQuestions.yaml b/walledeval/data/presets/walledai/ForbiddenQuestions.yaml
@@ -0,0 +1,3 @@
+name: walledai/ForbiddenQuestions
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/HarmBench.yaml b/walledeval/data/presets/walledai/HarmBench.yaml
@@ -0,0 +1,3 @@
+name: walledai/HarmBench
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/JailbreakBench.yaml b/walledeval/data/presets/walledai/JailbreakBench.yaml
@@ -0,0 +1,3 @@
+name: walledai/JailbreakBench
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/JailbreakHub.yaml b/walledeval/data/presets/walledai/JailbreakHub.yaml
@@ -0,0 +1,3 @@
+name: walledai/JailbreakHub
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/MaliciousInstruct.yaml b/walledeval/data/presets/walledai/MaliciousInstruct.yaml
@@ -0,0 +1,3 @@
+name: walledai/MaliciousInstruct
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/RTP.yaml b/walledeval/data/presets/walledai/RTP.yaml
@@ -0,0 +1,3 @@
+name: walledai/RTP
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/SGSafetyQuestions.yaml b/walledeval/data/presets/walledai/SGSafetyQuestions.yaml
@@ -0,0 +1,3 @@
+name: walledai/SGSafetyQuestions
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/SafeText.yaml b/walledeval/data/presets/walledai/SafeText.yaml
@@ -0,0 +1,3 @@
+name: walledai/SafeText
+split: train
+type: AutocompletePrompt
diff --git a/walledeval/data/presets/walledai/SafetyDatasets.yaml b/walledeval/data/presets/walledai/SafetyDatasets.yaml
@@ -0,0 +1,3 @@
+name: walledai/SafetyDatasets
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/SaladBench.yaml b/walledeval/data/presets/walledai/SaladBench.yaml
@@ -0,0 +1,3 @@
+name: walledai/SaladBench
+split: train
+type: MultipleResponseQuestion
diff --git a/walledeval/data/presets/walledai/SimpleSafetyTests.yaml b/walledeval/data/presets/walledai/SimpleSafetyTests.yaml
@@ -0,0 +1,5 @@
+name: walledai/SimpleSafetyTests
+split:
+  - info
+  - instruct
+type: Prompt
diff --git a/walledeval/data/presets/walledai/StrongREJECT.yaml b/walledeval/data/presets/walledai/StrongREJECT.yaml
@@ -0,0 +1,3 @@
+name: walledai/StrongREJECT
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/TDC23-RedTeaming.yaml b/walledeval/data/presets/walledai/TDC23-RedTeaming.yaml
@@ -0,0 +1,3 @@
+name: walledai/TDC23-RedTeaming
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/TET.yaml b/walledeval/data/presets/walledai/TET.yaml
@@ -0,0 +1,3 @@
+name: walledai/TET
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/WMDP.yaml b/walledeval/data/presets/walledai/WMDP.yaml
@@ -0,0 +1,6 @@
+name: walledai/WMDP
+split:
+  - bio
+  - chem
+  - cyber
+type: MultipleChoiceQuestion
diff --git a/walledeval/data/presets/walledai/WildGuardTest.yaml b/walledeval/data/presets/walledai/WildGuardTest.yaml
@@ -0,0 +1,3 @@
+name: walledai/WildGuardTest
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/WildJailbreak.yaml b/walledeval/data/presets/walledai/WildJailbreak.yaml
@@ -0,0 +1,3 @@
+name: walledai/WildJailbreak
+split: train
+type: Prompt
diff --git a/walledeval/data/presets/walledai/XSTest.yaml b/walledeval/data/presets/walledai/XSTest.yaml
@@ -0,0 +1,3 @@
+name: walledai/XSTest
+split: test
+type: Prompt