meta-llama3-8b-fine-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4b43de97-6a76-463e-a10d-2071fd563f0a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting protobuf\n",
      "  Downloading protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl.metadata (592 bytes)\n",
      "Collecting scikit-learn\n",
      "  Downloading scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
      "Collecting scipy\n",
      "  Downloading scipy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.19.5 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.26.3)\n",
      "Collecting joblib>=1.2.0 (from scikit-learn)\n",
      "  Downloading joblib-1.4.0-py3-none-any.whl.metadata (5.4 kB)\n",
      "Collecting threadpoolctl>=2.0.0 (from scikit-learn)\n",
      "  Downloading threadpoolctl-3.4.0-py3-none-any.whl.metadata (13 kB)\n",
      "Downloading protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl (302 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.8/302.8 kB\u001b[0m \u001b[31m23.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m216.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading scipy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.6/38.6 MB\u001b[0m \u001b[31m158.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading joblib-1.4.0-py3-none-any.whl (301 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.2/301.2 kB\u001b[0m \u001b[31m108.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading threadpoolctl-3.4.0-py3-none-any.whl (17 kB)\n",
      "Installing collected packages: threadpoolctl, scipy, protobuf, joblib, scikit-learn\n",
      "Successfully installed joblib-1.4.0 protobuf-5.26.1 scikit-learn-1.4.2 scipy-1.13.0 threadpoolctl-3.4.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting wandb\n",
      "  Downloading wandb-0.16.6-py3-none-any.whl.metadata (10 kB)\n",
      "Requirement already satisfied: Click!=8.0.0,>=7.1 in /opt/conda/lib/python3.10/site-packages (from wandb) (8.1.7)\n",
      "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
      "  Downloading GitPython-3.1.43-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: requests<3,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (2.31.0)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (5.9.0)\n",
      "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
      "  Downloading sentry_sdk-1.45.0-py2.py3-none-any.whl.metadata (9.9 kB)\n",
      "Collecting docker-pycreds>=0.4.0 (from wandb)\n",
      "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl.metadata (1.8 kB)\n",
      "Requirement already satisfied: PyYAML in /opt/conda/lib/python3.10/site-packages (from wandb) (6.0.1)\n",
      "Collecting setproctitle (from wandb)\n",
      "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.9 kB)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from wandb) (68.2.2)\n",
      "Collecting appdirs>=1.4.3 (from wandb)\n",
      "  Downloading appdirs-1.4.4-py2.py3-none-any.whl.metadata (9.0 kB)\n",
      "Collecting protobuf!=4.21.0,<5,>=3.19.0 (from wandb)\n",
      "  Downloading protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl.metadata (541 bytes)\n",
      "Requirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
      "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2024.2.2)\n",
      "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n",
      "Downloading wandb-0.16.6-py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m39.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n",
      "Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
      "Downloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m58.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl (294 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m294.6/294.6 kB\u001b[0m \u001b[31m77.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading sentry_sdk-1.45.0-py2.py3-none-any.whl (267 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m267.1/267.1 kB\u001b[0m \u001b[31m74.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
      "Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
      "Installing collected packages: appdirs, smmap, setproctitle, sentry-sdk, protobuf, docker-pycreds, gitdb, GitPython, wandb\n",
      "  Attempting uninstall: protobuf\n",
      "    Found existing installation: protobuf 5.26.1\n",
      "    Uninstalling protobuf-5.26.1:\n",
      "      Successfully uninstalled protobuf-5.26.1\n",
      "Successfully installed GitPython-3.1.43 appdirs-1.4.4 docker-pycreds-0.4.0 gitdb-4.0.11 protobuf-4.25.3 sentry-sdk-1.45.0 setproctitle-1.3.3 smmap-5.0.1 wandb-0.16.6\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mToken has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to /root/.cache/huggingface/token\n",
      "Login successful\n",
      "Requirement already satisfied: transformers[sentencepiece] in /opt/conda/lib/python3.10/site-packages (4.40.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (3.13.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.22.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (1.26.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (23.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (2024.4.16)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.19.1)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (0.4.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (4.65.0)\n",
      "Collecting sentencepiece!=0.1.92,>=0.1.91 (from transformers[sentencepiece])\n",
      "  Downloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
      "Requirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from transformers[sentencepiece]) (4.25.3)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers[sentencepiece]) (2024.2.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers[sentencepiece]) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers[sentencepiece]) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers[sentencepiece]) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers[sentencepiece]) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers[sentencepiece]) (2024.2.2)\n",
      "Downloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m27.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: sentencepiece\n",
      "Successfully installed sentencepiece-0.2.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (0.2.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "The following NEW packages will be installed:\n",
      "  git-lfs\n",
      "0 upgraded, 1 newly installed, 0 to remove and 23 not upgraded.\n",
      "Need to get 3503 kB of archives.\n",
      "After this operation, 10.4 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 git-lfs amd64 3.0.2-1ubuntu0.2 [3503 kB]\n",
      "Fetched 3503 kB in 1s (4882 kB/s)  \n",
      "debconf: delaying package configuration, since apt-utils is not installed\n",
      "Selecting previously unselected package git-lfs.\n",
      "(Reading database ... 16755 files and directories currently installed.)\n",
      "Preparing to unpack .../git-lfs_3.0.2-1ubuntu0.2_amd64.deb ...\n",
      "Unpacking git-lfs (3.0.2-1ubuntu0.2) ...\n",
      "Setting up git-lfs (3.0.2-1ubuntu0.2) ...\n",
      "Collecting tensorboardX\n",
      "  Downloading tensorboardX-2.6.2.2-py2.py3-none-any.whl.metadata (5.8 kB)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from tensorboardX) (1.26.3)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from tensorboardX) (23.1)\n",
      "Requirement already satisfied: protobuf>=3.20 in /opt/conda/lib/python3.10/site-packages (from tensorboardX) (4.25.3)\n",
      "Downloading tensorboardX-2.6.2.2-py2.py3-none-any.whl (101 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.7/101.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: tensorboardX\n",
      "Successfully installed tensorboardX-2.6.2.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install protobuf scikit-learn scipy\n",
    "!export CUDA_HOME=cuda-12.2\n",
    "!pip install wandb\n",
    "!pip install -q accelerate  bitsandbytes  trl==0.4.7\n",
    "!huggingface-cli login --token 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\n",
    "!pip install  transformers[sentencepiece]\n",
    "!pip install sentencepiece\n",
    "!apt-get install git-lfs\n",
    "!pip install tensorboardX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "19bd94ce-13fa-4115-8f04-11ad3bb3aba2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: accelerate==0.29.3 in /opt/conda/lib/python3.10/site-packages (0.29.3)\n",
      "Collecting peft==0.10.0\n",
      "  Downloading peft-0.10.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting bitsandbytes==0.40.2\n",
      "  Downloading bitsandbytes-0.40.2-py3-none-any.whl.metadata (9.8 kB)\n",
      "Requirement already satisfied: transformers==4.40.0 in /opt/conda/lib/python3.10/site-packages (4.40.0)\n",
      "Collecting trl==0.8.5\n",
      "  Downloading trl-0.8.5-py3-none-any.whl.metadata (11 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (1.26.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (23.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (5.9.0)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (2.2.1)\n",
      "Requirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (0.22.2)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.29.3) (0.4.3)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from peft==0.10.0) (4.65.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.40.0) (3.13.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.40.0) (2024.4.16)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.40.0) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/conda/lib/python3.10/site-packages (from transformers==4.40.0) (0.19.1)\n",
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from trl==0.8.5) (2.19.0)\n",
      "Collecting tyro>=0.5.11 (from trl==0.8.5)\n",
      "  Downloading tyro-0.8.3-py3-none-any.whl.metadata (7.9 kB)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate==0.29.3) (2024.2.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate==0.29.3) (4.9.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.29.3) (1.12)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.29.3) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.29.3) (3.1.3)\n",
      "Collecting docstring-parser>=0.14.1 (from tyro>=0.5.11->trl==0.8.5)\n",
      "  Downloading docstring_parser-0.16-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting rich>=11.1.0 (from tyro>=0.5.11->trl==0.8.5)\n",
      "  Downloading rich-13.7.1-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting shtab>=1.5.6 (from tyro>=0.5.11->trl==0.8.5)\n",
      "  Downloading shtab-1.7.1-py3-none-any.whl.metadata (7.3 kB)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (16.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (0.3.8)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (2.2.2)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (0.70.16)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.8.5) (3.9.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.40.0) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.40.0) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.40.0) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.40.0) (2024.2.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.8.5) (4.0.3)\n",
      "Collecting markdown-it-py>=2.2.0 (from rich>=11.1.0->tyro>=0.5.11->trl==0.8.5)\n",
      "  Downloading markdown_it_py-3.0.0-py3-none-any.whl.metadata (6.9 kB)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl==0.8.5) (2.15.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate==0.29.3) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.8.5) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.8.5) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.8.5) (2024.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate==0.29.3) (1.3.0)\n",
      "Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl==0.8.5)\n",
      "  Downloading mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets->trl==0.8.5) (1.16.0)\n",
      "Downloading peft-0.10.0-py3-none-any.whl (199 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.1/199.1 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading bitsandbytes-0.40.2-py3-none-any.whl (92.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.5/92.5 MB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading trl-0.8.5-py3-none-any.whl (245 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.1/245.1 kB\u001b[0m \u001b[31m82.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tyro-0.8.3-py3-none-any.whl (102 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.0/102.0 kB\u001b[0m \u001b[31m45.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading docstring_parser-0.16-py3-none-any.whl (36 kB)\n",
      "Downloading rich-13.7.1-py3-none-any.whl (240 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.7/240.7 kB\u001b[0m \u001b[31m81.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading shtab-1.7.1-py3-none-any.whl (14 kB)\n",
      "Downloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.5/87.5 kB\u001b[0m \u001b[31m37.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
      "Installing collected packages: bitsandbytes, shtab, mdurl, docstring-parser, markdown-it-py, rich, tyro, trl, peft\n",
      "  Attempting uninstall: bitsandbytes\n",
      "    Found existing installation: bitsandbytes 0.43.1\n",
      "    Uninstalling bitsandbytes-0.43.1:\n",
      "      Successfully uninstalled bitsandbytes-0.43.1\n",
      "  Attempting uninstall: trl\n",
      "    Found existing installation: trl 0.4.7\n",
      "    Uninstalling trl-0.4.7:\n",
      "      Successfully uninstalled trl-0.4.7\n",
      "Successfully installed bitsandbytes-0.40.2 docstring-parser-0.16 markdown-it-py-3.0.0 mdurl-0.1.2 peft-0.10.0 rich-13.7.1 shtab-1.7.1 trl-0.8.5 tyro-0.8.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -U accelerate==0.29.3 peft==0.10.0 bitsandbytes==0.40.2 transformers==4.40.0 trl==0.8.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "72b22b7d-9144-47f4-90f5-f2190e9b9fec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU name: NVIDIA A100-SXM4-80GB\n",
      "GPU model: NVIDIA A100-SXM4-80GB\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print(\"GPU name:\", torch.cuda.get_device_name(0))\n",
    "    print(\"GPU model:\", torch.cuda.get_device_name(0))\n",
    "else:\n",
    "    print(\"No GPU available\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4526cde9-8b9b-4917-bb50-d1dd284de4dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    HfArgumentParser,\n",
    "    TrainingArguments,\n",
    "    pipeline,\n",
    "    logging,\n",
    ")\n",
    "from peft import LoraConfig, PeftModel\n",
    "from trl import SFTTrainer\n",
    "hf_token = \"hf_....\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d9985edf-667a-4326-81c1-f5526bf34375",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The model that you want to train from the Hugging Face hub\n",
    "model_name = \"NousResearch/Meta-Llama-3-8B\"\n",
    "\n",
    "# The instruction dataset to use\n",
    "dataset_name = \"Hemanth-thunder/tamil-open-instruct-v1\"\n",
    "\n",
    "# Fine-tuned model name\n",
    "new_model = \"llama-3-8b-tamil-open-instruct-v1\"\n",
    "\n",
    "################################################################################\n",
    "# QLoRA parameters\n",
    "################################################################################\n",
    "\n",
    "# LoRA attention dimension\n",
    "lora_r = 64\n",
    "\n",
    "# Alpha parameter for LoRA scaling\n",
    "lora_alpha = 16\n",
    "\n",
    "# Dropout probability for LoRA layers\n",
    "lora_dropout = 0\n",
    "\n",
    "################################################################################\n",
    "# bitsandbytes parameters\n",
    "################################################################################\n",
    "\n",
    "# Activate 4-bit precision base model loading\n",
    "use_4bit = True\n",
    "\n",
    "# Compute dtype for 4-bit base models\n",
    "bnb_4bit_compute_dtype = \"float16\"\n",
    "\n",
    "# Quantization type (fp4 or nf4)\n",
    "bnb_4bit_quant_type = \"nf4\"\n",
    "\n",
    "# Activate nested quantization for 4-bit base models (double quantization)\n",
    "use_nested_quant = False\n",
    "\n",
    "################################################################################\n",
    "# TrainingArguments parameters\n",
    "################################################################################\n",
    "\n",
    "# Output directory where the model predictions and checkpoints will be stored\n",
    "output_dir = \"./results\"\n",
    "\n",
    "# Number of training epochs\n",
    "num_train_epochs = 1\n",
    "\n",
    "# Enable fp16/bf16 training (set bf16 to True with an A100)\n",
    "fp16 = False\n",
    "bf16 = False\n",
    "\n",
    "# Batch size per GPU for training\n",
    "per_device_train_batch_size = 16\n",
    "\n",
    "# Batch size per GPU for evaluation\n",
    "per_device_eval_batch_size = 16\n",
    "\n",
    "# Number of update steps to accumulate the gradients for\n",
    "gradient_accumulation_steps = 2\n",
    "\n",
    "# Enable gradient checkpointing\n",
    "gradient_checkpointing = True\n",
    "\n",
    "# Maximum gradient normal (gradient clipping)\n",
    "max_grad_norm = 0.3\n",
    "\n",
    "# Initial learning rate (AdamW optimizer)\n",
    "learning_rate = 2e-4\n",
    "\n",
    "# Weight decay to apply to all layers except bias/LayerNorm weights\n",
    "weight_decay = 0.001\n",
    "\n",
    "# Optimizer to use\n",
    "optim = \"paged_adamw_32bit\"\n",
    "\n",
    "# Learning rate schedule\n",
    "lr_scheduler_type = \"cosine\"\n",
    "\n",
    "# Number of training steps (overrides num_train_epochs)\n",
    "max_steps = -1\n",
    "\n",
    "# Ratio of steps for a linear warmup (from 0 to learning rate)\n",
    "warmup_ratio = 0.03\n",
    "\n",
    "# Group sequences into batches with same length\n",
    "# Saves memory and speeds up training considerably\n",
    "group_by_length = True\n",
    "\n",
    "# Save checkpoint every X updates steps\n",
    "save_steps = 0\n",
    "\n",
    "# Log every X updates steps\n",
    "logging_steps = 25\n",
    "\n",
    "################################################################################\n",
    "# SFT parameters\n",
    "################################################################################\n",
    "\n",
    "# Maximum sequence length to use\n",
    "max_seq_length = None\n",
    "\n",
    "# Pack multiple short examples in the same input sequence to increase efficiency\n",
    "packing = False\n",
    "\n",
    "# Load the entire model on the GPU 0\n",
    "device_map = {\"\": 0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f3b93ec1-47a0-45e9-926b-0bcae1c0fa12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cb2947ca5674633bcd4956b02a6f762",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/769 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c03466cc5ca14cecb68d00678c625e19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/95.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f138e25a4ddc4c479f72c47048bdd1e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/109M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa598a47c45446178d152f7e1ac90afc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/140M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a6a299d898c4e94bf3de83ea6e90940",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/204M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97afe3d303cd4c5da29861cb7ce1e0fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/493813 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load dataset (you can process it here)\n",
    "dataset = load_dataset(dataset_name, split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "adcefe75-8f08-4bad-8a9d-14a2a8a26507",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sample_dataset = dataset.shuffle().select(range(10000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6dd0c0e5-d31b-469e-b6b4-1da7358d991b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['instruction', 'output', 'input', 'text'],\n",
       "    num_rows: 10000\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sample_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bf647af-06a1-4934-a27e-df9b3547389c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "சரியான பதிலுடன் வேலையை வெற்றிகரமாக முடிக்க. தேவையான தகவலை உள்ளிடவும்.\n",
      "\n",
      "### Instruction:\n",
      "அனைத்து செல்லுபடியாகும் முக்கோண வகைகளின் பட்டியலை உருவாக்கவும்.\n",
      "\n",
      "### Response:\n",
      "செல்லுபடியாகும் முக்கோண வகைகள் சமபக்க, ஐசோசெல்ஸ், ஸ்கேலேன், வலது கோணம், மழுங்கிய மற்றும் கடுமையானவை.\n"
     ]
    }
   ],
   "source": [
    "#print(sample_dataset['text'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0bfbb931-68dc-42fc-bcfc-e2e0fd9277fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c679ad44da5d450f88afbfc733c3f91f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/654 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afa89399f73f42aeb1f7413211b7d05c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c17f73df62f43899fbda9e68bef4782",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d28c2ccb5ab64418b153312ff5310b4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00004.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93f2dd05cd8042439283e129216bf942",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00004.safetensors:   0%|          | 0.00/5.00G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e911f5950a60463cacdfbc0742abe8db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00003-of-00004.safetensors:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05bbd3d03e264787a9e33204ad9041c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00004-of-00004.safetensors:   0%|          | 0.00/1.17G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e20296e9a5ce4ec99d3b63bf8c0f7946",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1566ad82ca844d4292b9bf9ff6eac600",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/126 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are calling `save_pretrained` to a 4-bit converted model, but your `bitsandbytes` version doesn't support it. If you want to save 4-bit models, make sure to have `bitsandbytes>=0.41.3` installed.\n"
     ]
    }
   ],
   "source": [
    "# Load tokenizer and model with QLoRA configuration\n",
    "compute_dtype = getattr(torch, bnb_4bit_compute_dtype)\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=use_4bit,\n",
    "    bnb_4bit_quant_type=bnb_4bit_quant_type,\n",
    "    bnb_4bit_compute_dtype=compute_dtype,\n",
    "    bnb_4bit_use_double_quant=use_nested_quant,\n",
    ")\n",
    "\n",
    " \n",
    "\n",
    "# Load base model\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    quantization_config=bnb_config,\n",
    "    device_map=device_map,token=hf_token\n",
    ")\n",
    "model.config.use_cache = False\n",
    "model.config.pretraining_tp = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "afafb0bd-29bc-4800-ae50-1bf7bcbeff97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85e62fc521b84120b0168e28f2a8d50e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/50.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8a67eb097ec4a02bb62578230472760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/9.08M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72c68a7024e84e41971c2c2bfed06dea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/73.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "# Load LLaMA tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,token=hf_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1da6f461-1cec-4500-9022-11ed50875f93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c708e77741c84871af07f2944048bf56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/493813 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Must add EOS_TOKEN at response last line\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "EOS_TOKEN = tokenizer.eos_token \n",
    "def prompt_eos(sample):\n",
    "    sample['text'] = sample['text']+EOS_TOKEN\n",
    "    return sample\n",
    "dataset = dataset.map(prompt_eos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c197b380-2c70-4c6a-a4a9-97fbe62d7b65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load LoRA configuration\n",
    "peft_config = LoraConfig(\n",
    "    lora_alpha=lora_alpha,\n",
    "    lora_dropout=lora_dropout,\n",
    "    r=lora_r,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "\n",
    "# Set training parameters\n",
    "training_arguments = TrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
    "    optim=optim,\n",
    "    save_steps=save_steps,\n",
    "    logging_steps=logging_steps,\n",
    "    learning_rate=learning_rate,\n",
    "    weight_decay=weight_decay,\n",
    "    fp16=fp16,\n",
    "    bf16=bf16,\n",
    "    max_grad_norm=max_grad_norm,\n",
    "    max_steps=max_steps,\n",
    "    warmup_ratio=warmup_ratio,\n",
    "    group_by_length=group_by_length,\n",
    "    lr_scheduler_type=lr_scheduler_type,\n",
    "    report_to=\"tensorboard\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c6d21207-8de4-4b98-bf1b-66f2fa43456d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/trl/trainer/sft_trainer.py:246: UserWarning: You didn't pass a `max_seq_length` argument to the SFTTrainer, this will default to 1024\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ae0cfb049164ab8a17ac89e00f951f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/493813 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set supervised fine-tuning parameters\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset,\n",
    "    peft_config=peft_config,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=max_seq_length,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_arguments,\n",
    "    packing=packing,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12168034-3c16-40f9-9144-29cb0890ab50",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "369ca860-5669-47dd-ba34-f3808713afa7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "7d1fa166-3fd6-4f8e-8da2-23c210c5e343",
   "metadata": {},
   "outputs": [],
   "source": [
    " def generate_response(prompt, model):\n",
    "    encoded_input = tokenizer(prompt,  return_tensors=\"pt\", add_special_tokens=True)\n",
    "    model_inputs = encoded_input.to('cuda')\n",
    "    \n",
    "    generated_ids = model.generate(**model_inputs, max_new_tokens=512, do_sample=True, pad_token_id=tokenizer.eos_token_id)\n",
    "    \n",
    "    decoded_output = tokenizer.batch_decode(generated_ids)\n",
    "    \n",
    "    return decoded_output[0].replace(prompt, \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "098c5a4c-5de1-4062-8ec2-dad8e5784613",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' இ-காமர்ஸ் இணையதளத்தை உருவாக்குவதற்கான செலவை மதிப்பிடவும்.'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_dataset['instruction'][5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0d233579-fefe-4234-ba46-a5c165bdc300",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_template_wo_input=\"\"\"சரியான பதிலுடன் வேலையை வெற்றிகரமாக முடிக்க, தேவையான தகவலை உள்ளிடவும்.\n",
    "\n",
    "### Instruction:\n",
    "{}\n",
    "\n",
    "### Response:\"\"\"\n",
    "\n",
    "# with Input\n",
    "prompt_template_input=\"\"\"சரியான பதிலுடன் வேலையை வெற்றிகரமாக முடிக்க, வழங்கப்பட்ட வழிகாட்டுதல்களைப் பின்பற்றி, தேவையான தகவலை உள்ளிடவும்.\n",
    "\n",
    "### Instruction:\n",
    "{}\n",
    "\n",
    "### Input:\n",
    "{}\n",
    "\n",
    "### Response:\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "fb03cea4-c1a2-4842-8f74-f70749687936",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def create_prompt(sample):\n",
    "#     if 'nan' not in sample['input']:\n",
    "#       prompt = prompt_template_input.format(sample['instruction'][-1].strip(),sample['input'])\n",
    "#     else:\n",
    "#      prompt = prompt_template_wo_input.format(sample['instruction'][-1].strip())\n",
    "#     return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "aec4efd4-e44f-47b9-8ba7-6d3610b6bcef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' \\nகாசோலிக்கு எவ்வளவு பணம் சேமிக்க வேண்டும் என்பது எங்களுக்கு முக்கியமானது, ஏனென்றால் பொருளாதார நிகழ்வுகளால் அடிப்படையான சேமிப்பு எங்கள் வாங்க முடிவு செய்ய எங்கள் முயற்சியை வழிநடத்துகிறது. காசோலை, உங்கள் காசோலைக் கடந்த 6 மாதங்களில் வரும் ஈடுபடுத்துதல் இல்லாதவையும் படிமங்களை மட்டும் கடப்பாக உங்களுக்கு பயன்படுத்துவதில் முதல் படிக்கு உங்களை அலுவலகங்களுக்கு வசதியாக்க வேண்டும்'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt_template_wo_input=\"\"\"சரியான பதிலுடன் வேலையை வெற்றிகரமாக முடிக்க, தேவையான தகவலை உள்ளிடவும்.\n",
    "\n",
    "### Instruction:\n",
    "இ-காமர்ஸ் இணையதளத்தை உருவாக்குவதற்கான செலவை மதிப்பிடவும்.\n",
    "\n",
    "### Response:\"\"\"\n",
    "generate_response(prompt, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d485e3a8-0596-4c9c-a8b9-c12004f9ea55",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}