From 736f31e1886c465e3339544dcd0e1da09a6ff266 Mon Sep 17 00:00:00 2001 From: malinjawi Date: Sat, 12 Oct 2024 15:32:02 +0300 Subject: [PATCH 1/2] refactor gemma evaluation --- recipes/configs/gemma/evaluation.yaml | 40 +++++++++++++++++++++++++++ torchtune/_recipe_registry.py | 4 +++ 2 files changed, 44 insertions(+) create mode 100644 recipes/configs/gemma/evaluation.yaml diff --git a/recipes/configs/gemma/evaluation.yaml b/recipes/configs/gemma/evaluation.yaml new file mode 100644 index 0000000000..fa0e0eebc6 --- /dev/null +++ b/recipes/configs/gemma/evaluation.yaml @@ -0,0 +1,40 @@ +# Config for EleutherEvalRecipe in eleuther_eval.py +# +# To launch, run the following command: +# tune run eleuther_eval --config gemma/evaluation + +# Model Arguments +model: + _component_: torchtune.models.gemma.gemma_2b + +# Checkpointer +checkpointer: + _component_: torchtune.training.FullModelHFCheckpointer + checkpoint_dir: /tmp/gemma-2b + checkpoint_files: [ + model-00001-of-00002.safetensors, + model-00002-of-00002.safetensors, + ] + #recipe_checkpoint: null + output_dir: ./ # Not needed + model_type: GEMMA + +# Tokenizer +tokenizer: + _component_: torchtune.models.gemma.gemma_tokenizer + path: /tmp/gemma-2b//tokenizer.model + +# Environment +device: gpu +dtype: bf16 +seed: 1234 # It is not recommended to change this seed, b/c it matches EleutherAI's default seed + +# EleutherAI specific eval args +tasks: ["truthfulqa_mc2"] +limit: null +max_seq_length: 4096 +batch_size: 8 +enable_kv_cache: True + +# Quantization specific args +quantizer: null diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py index ca1ce6150e..30ba7b3c81 100644 --- a/torchtune/_recipe_registry.py +++ b/torchtune/_recipe_registry.py @@ -328,6 +328,10 @@ class Recipe: name="qwen2/evaluation", file_path="qwen2/evaluation.yaml", ), + Config( + name="gemma/evaluation", + file_path="gemma/evaluation.yaml", + ), ], supports_distributed=False, ), From d1d12338ec7cf03444611910b9b5761e6698d52d Mon Sep 17 00:00:00 2001 From: malinjawi Date: Sat, 12 Oct 2024 22:41:05 +0300 Subject: [PATCH 2/2] cleaned up code for comments --- recipes/configs/gemma/evaluation.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/recipes/configs/gemma/evaluation.yaml b/recipes/configs/gemma/evaluation.yaml index fa0e0eebc6..2ff8f78546 100644 --- a/recipes/configs/gemma/evaluation.yaml +++ b/recipes/configs/gemma/evaluation.yaml @@ -15,17 +15,16 @@ checkpointer: model-00001-of-00002.safetensors, model-00002-of-00002.safetensors, ] - #recipe_checkpoint: null output_dir: ./ # Not needed model_type: GEMMA # Tokenizer tokenizer: _component_: torchtune.models.gemma.gemma_tokenizer - path: /tmp/gemma-2b//tokenizer.model + path: /tmp/gemma-2b/tokenizer.model # Environment -device: gpu +device: cuda dtype: bf16 seed: 1234 # It is not recommended to change this seed, b/c it matches EleutherAI's default seed