Fix the IndexError in get_response

DataCollator.py

@@ -0,0 +1,94 @@
+from dataclasses import dataclass
+from transformers.data.data_collator import *
+
+@dataclass
+class DataCollatorForSeq2Seq:
+    """
+    Data collator that will dynamically pad the inputs received, as well as the labels.
+
+    Args:
+        tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]):
+            The tokenizer used for encoding the data.
+        model ([`PreTrainedModel`]):
+            The model that is being trained. If set and has the *prepare_decoder_input_ids_from_labels*, use it to
+            prepare the *decoder_input_ids*
+
+            This is useful when using *label_smoothing* to avoid calculating loss twice.
+        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
+            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
+            among:
+
+            - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single
+              sequence is provided).
+            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+              acceptable input length for the model if that argument is not provided.
+            - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different lengths).
+        max_length (`int`, *optional*):
+            Maximum length of the returned list and optionally padding length (see above).
+        pad_to_multiple_of (`int`, *optional*):
+            If set will pad the sequence to a multiple of the provided value.
+
+            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
+            7.5 (Volta).
+        label_pad_token_id (`int`, *optional*, defaults to -100):
+            The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).
+        return_tensors (`str`):
+            The type of Tensor to return. Allowable values are "np", "pt" and "tf".
+    """
+
+    tokenizer: PreTrainedTokenizerBase
+    model: Optional[Any] = None
+    padding: Union[bool, str, PaddingStrategy] = True
+    max_length: Optional[int] = None
+    pad_to_multiple_of: Optional[int] = None
+    label_pad_token_id: int = -100
+    return_tensors: str = "pt"
+
+    def __call__(self, features, return_tensors=None):
+        if return_tensors is None:
+            return_tensors = self.return_tensors
+        labels = [feature["labels"] for feature in features] if "labels" in features[0].keys() else None
+        # We have to pad the labels before calling `tokenizer.pad` as this method won't pad them and needs them of the
+        # same length to return tensors.
+        if labels is not None:
+            max_label_length = max(len(l) for l in labels)
+            if self.pad_to_multiple_of is not None:
+                max_label_length = (
+                    (max_label_length + self.pad_to_multiple_of - 1)
+                    // self.pad_to_multiple_of
+                    * self.pad_to_multiple_of
+                )
+
+            padding_side = self.tokenizer.padding_side
+            for feature in features:
+                remainder = [self.label_pad_token_id] * (max_label_length - len(feature["labels"]))
+                if isinstance(feature["labels"], list):
+                    feature["labels"] = (
+                        feature["labels"] + remainder if padding_side == "right" else remainder + feature["labels"]
+                    )
+                elif padding_side == "right":
+                    feature["labels"] = np.concatenate([feature["labels"], remainder]).astype(np.int64)
+                else:
+                    feature["labels"] = np.concatenate([remainder, feature["labels"]]).astype(np.int64)
+
+        features = self.tokenizer.pad(
+            features,
+            padding=self.padding,
+            max_length=self.max_length,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors=return_tensors,
+        )
+
+        # prepare decoder_input_ids
+        if (
+            labels is not None
+            and self.model is not None
+            and hasattr(self.model, "prepare_decoder_input_ids_from_labels")
+        ):
+            decoder_input_ids = self.model.prepare_decoder_input_ids_from_labels(labels=features["labels"])
+            features["decoder_input_ids"] = decoder_input_ids
+
+        print(features)
+        exit()
+
+        return features

README.md

@@ -1,3 +1,15 @@
+
+Forked from Alpaca-LoRA, modify some parts of the code for my cutomized usage.
+
+- add batched evaluation.
+- add metric calculation.
+- fix the error from `torch.complie` and the PEFT APIs. see [issue](https://github.com/tloen/alpaca-lora/issues/609)
+- ...
+
+
+
+
+
 # 🦙🌲🤏 Alpaca-LoRA
 
 - 🤗 **Try the pretrained model out [here](https://huggingface.co/spaces/tloen/alpaca-lora), courtesy of a GPU grant from Huggingface!**

calculate_metrics.py

@@ -0,0 +1,133 @@
+import argparse
+import copy
+import json
+import os
+
+import numpy as np
+
+from calculate_metrics_src import compute_grouped_metrics_v2, compute_metrics
+
+
+def process_superni(superni_preds, superni_meta):
+    assert len(superni_preds) == len(superni_meta), "The length of the predictions {} and the metadata {} should be the same".format(len(superni_preds), len(superni_meta))
+    final_res = []
+    for pred, meta in zip(superni_preds, superni_meta):
+        # to ensure the order are the same
+        assert pred["input"] == meta["input"], "The input of the prediction {} and the metadata {} should be the same".format(pred["input"], meta["input"])
+        assert pred["instruction"] == meta["instruction"], "The instruction of the prediction {} and the metadata {} should be the same".format(pred["instruction"], meta["instruction"])
+        item = copy.deepcopy(meta)
+        item["response"] = pred["response"]
+        final_res.append(item)
+
+    return final_res
+
+def calculate_metrics(all_results, save_path=None, save_prefix=None):
+    instructions, inputs, outputs, responses = [], [], [], []
+    categoreis = []
+    for result in all_results:
+        instruction = result["instruction"]
+        input = result["input"]
+        output = result["output"]
+        response = result["response"]
+        if "categories" in result:
+            # superni
+            categoreis.append(result["categories"])
+            assert type(output) == list, "The output of superni should be a list, but got {}, save_prefix: {}".format(output, save_prefix)
+            outputs.append(output) # the output of the superni is already a list
+        else:
+            # p3, mmlu, bbh
+            assert type(output) == str and type(output) != list, "The output of p3, mmlu, and bbh should be a string (only superni is a list), but got {}, save_prefix: {}".format(output, save_prefix)
+            outputs.append([output])  # we expect the item in the list to be a list (cuz the `metric_max_over_ground_truths`)
+
+        instructions.append(instruction)
+        inputs.append(input)
+        responses.append(response)
+
+    # calculate the metrics
+    if len(categoreis) == 0:
+        categoreis = None
+
+    metrics = compute_ni_metrics(responses, outputs, instructions, inputs, categories=categoreis, save_path=save_path, save_prefix=save_prefix)
+
+    return metrics
+
+
+def compute_ni_metrics(preds:list, references:list, instructions:list, inputs:list, categories=None, save_prefix=None, save_path=None):
+    decoded_preds = preds
+    result = compute_metrics(predictions=decoded_preds, references=references)
+    categories = ["_".join(it[0].lower().split()) for it in categories] if categories is not None else None
+    if categories is not None:
+        result_per_category = compute_grouped_metrics_v2(predictions=decoded_preds, references=references, groups=categories)
+        result.update(result_per_category)
+    prediction_lens = [len(pred.split()) for pred in decoded_preds]  # this gen len is different from the origin code, which is the word length instead of token length
+    result["gen_len"] = np.mean(prediction_lens)
+    result = {k: round(v, 4) for k, v in result.items()}
+
+    assert save_path is not None and save_prefix is not None, "The save_path and save_prefix should not be None"
+
+    if save_path is not None and save_prefix is not None:
+        with open(os.path.join(save_path, f"{save_prefix}_eval_predictions.jsonl"), "w") as fout:
+            for instruction, input, output, pred in zip(instructions, inputs, references, decoded_preds):
+                fout.write(json.dumps({
+                    "Definition": instruction,
+                    "Input": input,
+                    "Output": output,
+                    "Prediction": pred
+                }) + "\n")
+        # save the scores
+        with open(os.path.join(save_path, f"{save_prefix}_eval_scores.json"), "w") as fout:
+            json.dump(result, fout, indent=4)
+
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results_path",type=str,default="./alpaca_2")
+
+    args, unparsed = parser.parse_known_args()
+    if unparsed:
+        raise ValueError(unparsed)
+
+
+    # read and calculate the metrics on  all the four benchmarks
+    # superni
+    # note that the supernni should also read the 'superni_test_11810_eval_usage.json' file
+    # cuz we have to use the categories in that file to calcultae the grouped metrics
+    with open(os.path.join(args.results_path, "superni.json"), "r") as fin:
+        superni_preds = json.load(fin)
+
+    with open("/data/rml6079/projects/muffin_llama/alpaca-lora/eval_benchmarks/superni_test_11810_eval_usage.json", "r") as fin:
+        superni_meta = json.load(fin)
+
+    # combine these two files to get the final list to calculate the metrics.
+    # the only difference of superni_results, is that it has the response field, and the output of the superni is a list instead of a string
+    superni_results = process_superni(superni_preds, superni_meta)
+    calculate_metrics(superni_results, save_path=args.results_path, save_prefix="superni")
+    print("superni done")
+
+    # p3
+    with open(os.path.join(args.results_path, "p3.json"), "r") as fin:
+        p3_results = json.load(fin)
+    calculate_metrics(p3_results, save_path=args.results_path, save_prefix="p3")
+    print("p3 done")
+
+    # mmlu
+    with open(os.path.join(args.results_path, "mmlu.json"), "r") as fin:
+        mmlu_results = json.load(fin)
+    calculate_metrics(mmlu_results, save_path=args.results_path, save_prefix="mmlu")
+    print("mmlu done")
+
+    # bbh
+    with open(os.path.join(args.results_path, "bbh.json"), "r") as fin:
+        bbh_results = json.load(fin)
+    calculate_metrics(bbh_results, save_path=args.results_path, save_prefix="bbh")
+    print("bbh done")
+
+
+
+
+
+
+
+if __name__ == "__main__":
+    main()