commands_training_ultrafeedback_RainbowPO_final_run.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RainbowPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. RainbowPO beta 10, home advantage 0.3, mixing 0.25\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_10_hadv_0.3_mixing_0.25\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 10 \\\n",
    "    --loss_type 'mallows_dpo' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.25 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.3 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. RainbowPO beta 10, home advantage 0.1, mixing 0.25\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_10_hadv_0.1_mixing_0.25\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 10 \\\n",
    "    --loss_type 'mallows_dpo' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.25 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.1 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. RainbowPO beta 10, home advantage 0.1, mixing 0.5\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_10_hadv_0.1_mixing_0.5\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 10 \\\n",
    "    --loss_type 'mallows_dpo' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.5 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.1 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. RainbowPO beta 1, home advantage 0.3, mixing=0.5, square loss\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_1_hadv_0.3_mixing_0.5_square_loss\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 1 \\\n",
    "    --loss_type 'mallows_ipo' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.5 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.3 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5. RainbowPO beta 0.1, home advantage 0.3, mixing=0.5, square loss\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_0.1_hadv_0.3_mixing_0.5_square_loss\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 0.1 \\\n",
    "    --loss_type 'mallows_ipo' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.5 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.3 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. RainbowPO beta 0.1, home advantage 0.3, mixing=0.5, square hinge loss\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"/home/hanyang/RainbowPO/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1 \\\n",
    "    --learning_rate 1e-6 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --report_to wandb \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"/home/hanyang/RainbowPO/models_rlhf/Llama3-Instruct_armorm_rainbowpo_beta_0.1_hadv_0.3_mixing_0.5_square_hinge_loss\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 0.1 \\\n",
    "    --loss_type 'mallows_ipo_modified' \\\n",
    "    --if_mixing_alpha True \\\n",
    "    --mixing_alpha 0.5 \\\n",
    "    --length_normalization True \\\n",
    "    --home_advantage 0.3 \\\n",
    "    --num_train_epochs 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7. Baseline IPO retrain\n",
    "!accelerate launch --config_file=examples/accelerate_configs/deepspeed_zero3.yaml --num_processes 8 examples/scripts/dpo_zero3.py \\\n",
    "    --dataset_name=\"<some path>/trl/examples/datasets/UltraFeedback_armorm_trl\" \\\n",
    "    --model_name_or_path=\"<some path>/model_zoo/Meta-Llama-3-8B-Instruct\" \\\n",
    "    --per_device_train_batch_size 1\\\n",
    "    --per_device_eval_batch_size 1 \\\n",
    "    --learning_rate 5e-7 \\\n",
    "    --gradient_accumulation_steps 32 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 10 \\\n",
    "    --output_dir=\"<some path>/trl/models_rlhf/Llama3-Instruct_armorm_ipo_beta_0.005_lr_0.5_max_token_2048\" \\\n",
    "    --optim adamw_torch \\\n",
    "    --max_length 2048 \\\n",
    "    --max_prompt_length 1800 \\\n",
    "    --seed 42 \\\n",
    "    --bf16 \\\n",
    "    --warmup_ratio 0.1 \\\n",
    "    --logging_first_step \\\n",
    "    --no_remove_unused_columns \\\n",
    "    --attn_implementation 'flash_attention_2' \\\n",
    "    --beta 0.005 \\\n",
    "    --loss_type \"ipo\" \\\n",
    "    --num_train_epochs 1 \\\n",
    "    --if_mixing_alpha False \\\n",
    "    --length_normalization False \\\n",
    "    --home_advantage 0.0"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.1.-1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}