llm_recsys.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "97c3a78f-cfdc-4552-8d8b-a8351451318c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tqdm.notebook as tqdm\n",
    "import pickle\n",
    "from openai import OpenAI\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "from surprise import accuracy, Dataset, SVD, Prediction, Trainset\n",
    "from surprise.model_selection import train_test_split as s_train_test_split\n",
    "\n",
    "with open(\"openai.secret\") as f:\n",
    "    os.environ[\"OPENAI_API_KEY\"] = f.read().strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15093fe2-743e-4474-87fd-edcb28bbad10",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7e4ae31f-337e-4eea-96c9-a1151ca8665e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Rating = tuple[int | str, int | str, float]\n",
    "UserHistory = dict[int, float]\n",
    "\n",
    "ratings = Dataset.load_builtin(\"ml-100k\")\n",
    "train_ratings, test_ratings = s_train_test_split(ratings, test_size=0.2, random_state=0)\n",
    "train_mean_rating = np.mean([r for _, _, r in train_ratings.all_ratings()])\n",
    "\n",
    "\n",
    "def to_inner_user_id(raw_user_id: str) -> int | None:\n",
    "    try:\n",
    "        return train_ratings.to_inner_uid(raw_user_id)\n",
    "    except ValueError:\n",
    "        return None\n",
    "    \n",
    "    \n",
    "def to_inner_movie_id(raw_movie_id: str) -> int | None:\n",
    "    try:\n",
    "        return train_ratings.to_inner_iid(raw_movie_id)\n",
    "    except ValueError:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe8ec279-4ceb-427a-8f42-bd72f51bf3ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_classification(predictions: list[Prediction]) -> None:\n",
    "    y_true = [int(p.r_ui) for p in predictions]\n",
    "    y_pred = [round(p.est) for p in predictions]\n",
    "    print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e0d41c8-fb24-43f1-868d-097687420abd",
   "metadata": {},
   "source": [
    "## Baseline SVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b321cc2-beca-4078-90f9-a47305360803",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.9443\n",
      "MAE:  0.7441\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7441078856310742"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recsys = SVD()\n",
    "recsys.fit(train_ratings)\n",
    "\n",
    "baseline_predictions = recsys.test(test_ratings)\n",
    "\n",
    "accuracy.rmse(recsys.test(test_ratings))\n",
    "accuracy.mae(recsys.test(test_ratings))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7243daf3-c4bb-414a-85d8-629caad3b653",
   "metadata": {},
   "source": [
    "## Baseline Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9bb9c57f-441a-4a7b-aab8-7bd4e30118e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.8897\n",
      "MAE:  1.5142\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.51415"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_predictions = [\n",
    "    Prediction(\n",
    "        uid=raw_user_id,\n",
    "        iid=raw_movie_id,\n",
    "        r_ui=rating,\n",
    "        est=np.random.randint(1, 6),\n",
    "        details={},\n",
    "    )\n",
    "    for raw_user_id, raw_movie_id, rating  in test_ratings\n",
    "]\n",
    "\n",
    "accuracy.rmse(random_predictions)\n",
    "accuracy.mae(random_predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfa15ed1-6d88-4c62-99e9-914624152416",
   "metadata": {},
   "source": [
    "## LLM + SVD + RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "78375807-2298-4b70-9786-5f99cd1be238",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_user_item(ratings: list[Rating]) -> pd.DataFrame:\n",
    "    return (\n",
    "        pd\n",
    "        .DataFrame(ratings, columns=[\"user_id\", \"movie_id\", \"rating\"])\n",
    "        .groupby([\"user_id\", \"movie_id\"])\n",
    "        .agg(rating=(\"rating\", \"sum\"))\n",
    "        .reset_index()\n",
    "        .pivot(index=\"user_id\", columns=\"movie_id\", values=\"rating\")\n",
    "        .fillna(0)\n",
    "        .sort_index()\n",
    "    )\n",
    "\n",
    "\n",
    "def get_user_histories(user_item: pd.DataFrame) -> dict[int, UserHistory]:\n",
    "    return user_item.apply(lambda r: dict(r[r > 0].items()), axis=1).to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "92017ce9-7fee-496f-abb6-0898abed165e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# using inner ids from now on\n",
    "train_user_item = get_user_item(train_ratings.all_ratings())\n",
    "n_dims = 100\n",
    "u, s, v = np.linalg.svd(train_user_item, full_matrices=True)\n",
    "train_user_embeddings = u[:, :n_dims]\n",
    "train_user_histories = get_user_histories(train_user_item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "033b0472-b95a-4a14-9a35-76f55134ea05",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>NearestNeighbors(metric=&#x27;cosine&#x27;, n_neighbors=20)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">NearestNeighbors</label><div class=\"sk-toggleable__content\"><pre>NearestNeighbors(metric=&#x27;cosine&#x27;, n_neighbors=20)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "NearestNeighbors(metric='cosine', n_neighbors=20)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_knn = NearestNeighbors(n_neighbors=20, metric=\"cosine\")\n",
    "train_knn.fit(train_user_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "35701f7e-1e29-4554-9dde-dc950076056c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_nearest_user_histories(\n",
    "    user_id: int,\n",
    "    knn: NearestNeighbors,\n",
    "    user_embeddings: np.array,\n",
    "    user_histories: dict[int, UserHistory]\n",
    ") -> dict[int, UserHistory]:\n",
    "    \n",
    "    vec = user_embeddings[user_id, np.newaxis]\n",
    "    nearest_user_ids = knn.kneighbors(vec, return_distance=False)\n",
    "    return {\n",
    "        near_user_id: user_histories[near_user_id]\n",
    "        for near_user_id in nearest_user_ids[0].tolist()\n",
    "        if near_user_id != user_id\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f270abf3-497e-4d41-b42f-a957c3b25336",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movies_to_predict(\n",
    "    user_id: int,\n",
    "    test_ratings: list[Rating],\n",
    ") -> list[int]:\n",
    "\n",
    "    return [\n",
    "        to_inner_movie_id(raw_movie_id) for raw_user_id, raw_movie_id, _ in test_ratings\n",
    "        if to_inner_user_id(raw_user_id) == user_id and to_inner_movie_id(raw_movie_id) is not None\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8ac3d1e0-265d-41a2-9f00-40463843f131",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fmt_list(x: list[int]) -> str:\n",
    "    return str(x).strip(\"[]\")\n",
    "\n",
    "\n",
    "def fmt_user_history(user_history: UserHistory) -> str:\n",
    "    return \", \".join(\n",
    "        f\"{movie_id}: {rating}\" for movie_id, rating in user_history.items()\n",
    "    )\n",
    "\n",
    "\n",
    "def get_instruction(\n",
    "    user_history: UserHistory,\n",
    "    nearest_user_histories: dict[int, UserHistory],\n",
    "    movies_to_predict: list[int],\n",
    ") -> str:\n",
    "    \n",
    "    formatted_nearest_user_histories = \"\\n\".join(\n",
    "        f\"user {i}: {fmt_user_history(near_user_history)}.\"\n",
    "        for i, near_user_history\n",
    "        in enumerate(nearest_user_histories.values())\n",
    "    )\n",
    "    \n",
    "    return f\"\"\"\n",
    "        A user has seen and rated the following movies: {fmt_user_history(user_history)}.\n",
    "\n",
    "        Users with similar preferences have seen and rated these movies:\n",
    "        {formatted_nearest_user_histories}.\n",
    "\n",
    "        Each comma-separated value in the lists above represents a movie id followed\n",
    "        by a numerical user rating.\n",
    "\n",
    "        Use the data above to predict the ratings the user has given to\n",
    "        the movies: {fmt_list(movies_to_predict)}. Give exactly one rating\n",
    "        to each of the movies.\n",
    "        \n",
    "        Respond with just a list of ratings and nothing else. Your response needs\n",
    "        to have exactly {len(movies_to_predict)} ratings.\n",
    "    \"\"\".replace(\"    \", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fc06d6cd-9a94-4bf0-840e-5df5e12b7ca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_response(response: str) -> list[float]:\n",
    "    \n",
    "    strip = \".[]` \"\n",
    "    response = response.replace(\"plaintext\\n\", \"\")\n",
    "    if \"\\n\" in response:\n",
    "        numbers = (line.strip(strip)[-3:] for line in response.split(\"\\n\"))\n",
    "        numbers = filter(lambda n: n != \"\", numbers)\n",
    "    else:\n",
    "        numbers = response.strip(strip).split(\",\")\n",
    "        \n",
    "    return list(map(float, numbers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f9b9ec0a-0ca0-46a2-8143-4c380d67b34e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(instruction: str, model: str) -> list[float]:\n",
    "    completion = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"user\", \"content\": instruction}\n",
    "        ]\n",
    "    )\n",
    "    \n",
    "    response = completion.choices[0].message.content\n",
    "    try:\n",
    "        return parse_response(response)\n",
    "    except Exception:\n",
    "        return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "78cfa1d0-078e-4830-bb95-11573488cef2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_all(\n",
    "    model: str,\n",
    "    output_file_path: str,\n",
    "    train_user_item: pd.DataFrame,\n",
    "    train_user_histories: list[UserHistory],\n",
    "    train_user_embeddings: np.array,\n",
    "    train_knn: NearestNeighbors,\n",
    "    test_ratings: list[Rating],\n",
    ") -> list[Prediction]:\n",
    "    \n",
    "    client = OpenAI()\n",
    "\n",
    "    if not os.path.exists(output_file_path):\n",
    "        predictions = {}\n",
    "    else:\n",
    "        with open(output_file_path, \"rb\") as f:\n",
    "            predictions = pickle.load(f)\n",
    "\n",
    "    for user_id in tqdm.tqdm(train_user_item.index.tolist()):\n",
    "\n",
    "        if user_id in predictions:\n",
    "            continue\n",
    "\n",
    "        movies_to_predict = get_movies_to_predict(\n",
    "            user_id=user_id,\n",
    "            test_ratings=test_ratings,\n",
    "        )\n",
    "\n",
    "        if not movies_to_predict:\n",
    "            continue\n",
    "\n",
    "        user_history=train_user_histories[user_id]\n",
    "        nearest_user_histories=get_nearest_user_histories(\n",
    "            user_id=user_id,\n",
    "            knn=train_knn,\n",
    "            user_embeddings=train_user_embeddings,\n",
    "            user_histories=train_user_histories,\n",
    "        )\n",
    "\n",
    "        instruction = get_instruction(\n",
    "            user_history=user_history,\n",
    "            nearest_user_histories=nearest_user_histories,\n",
    "            movies_to_predict=movies_to_predict,\n",
    "        )\n",
    "\n",
    "        predicted_ratings = predict(instruction, model)\n",
    "        predictions[user_id] = {\n",
    "            \"predicted_ratings\": predicted_ratings,\n",
    "            \"movies_to_predict\": movies_to_predict\n",
    "        }\n",
    "\n",
    "        with open(output_file_path, \"wb\") as f:\n",
    "            pickle.dump(predictions, f)\n",
    "            \n",
    "\n",
    "    processed_predictions = {}\n",
    "    to_raw_user_id = train_ratings.to_raw_uid\n",
    "    to_raw_movie_id = train_ratings.to_raw_iid\n",
    "\n",
    "    for user_id in predictions.keys():\n",
    "        movies_to_predict = predictions[user_id][\"movies_to_predict\"]\n",
    "        predicted_ratings = predictions[user_id][\"predicted_ratings\"]\n",
    "\n",
    "        if not isinstance(predicted_ratings, list):\n",
    "            raise Exception(user_id)\n",
    "\n",
    "        len_movies = len(movies_to_predict)\n",
    "        len_pred = len(predicted_ratings)\n",
    "\n",
    "        predicted_ratings = [\n",
    "            r if r > 0 else train_mean_rating for r in predicted_ratings\n",
    "        ]\n",
    "\n",
    "        if len_movies > len_pred:\n",
    "            predicted_ratings += [train_mean_rating] * (len_movies - len_pred)\n",
    "\n",
    "        if len_pred > len_movies:\n",
    "            predicted_ratings = predicted_ratings[:len_movies]\n",
    "\n",
    "        processed_predictions[to_raw_user_id(user_id)] = {\n",
    "            to_raw_movie_id(movie_id): predicted_rating\n",
    "            for movie_id, predicted_rating in zip(movies_to_predict, predicted_ratings)\n",
    "        }\n",
    "\n",
    "\n",
    "    return [\n",
    "        Prediction(\n",
    "            uid=raw_user_id,\n",
    "            iid=raw_movie_id,\n",
    "            r_ui=rating,\n",
    "            est=processed_predictions[raw_user_id][raw_movie_id],\n",
    "            details={},\n",
    "        )\n",
    "        for raw_user_id, raw_movie_id, rating  in test_ratings\n",
    "        if raw_user_id in processed_predictions\n",
    "        and raw_movie_id in processed_predictions[raw_user_id]\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "df061807-ef37-4add-8fd0-a2b6b8a874a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6bb57964d44473997a4b54c2573df37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/943 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gpt_4o_predictions = predict_all(\n",
    "    model=\"gpt-4o\",\n",
    "    output_file_path=\"predictions_gpt-4o.pkl\",\n",
    "    train_user_item=train_user_item,\n",
    "    train_user_histories=train_user_histories,\n",
    "    train_user_embeddings=train_user_embeddings,\n",
    "    train_knn=train_knn,\n",
    "    test_ratings=test_ratings,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cd90a814-f459-4c57-ab1a-9a59cc2d3aa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76f2e2af431b41978994b06ea19eee2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/943 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gpt_4o_mini_predictions = predict_all(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    output_file_path=\"predictions_gpt-4o-mini.pkl\",\n",
    "    train_user_item=train_user_item,\n",
    "    train_user_histories=train_user_histories,\n",
    "    train_user_embeddings=train_user_embeddings,\n",
    "    train_knn=train_knn,\n",
    "    test_ratings=test_ratings,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4f42b06b-b980-43ea-8b9b-a231238d5201",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.2879\n",
      "MAE:  0.9611\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9611227528300942"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gpt 4o\n",
    "accuracy.rmse(gpt_4o_predictions)\n",
    "accuracy.mae(gpt_4o_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "id": "2874c529-c34f-4cde-8597-f607c89f59f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.3855\n",
      "MAE:  1.0230\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0229663394109396"
      ]
     },
     "execution_count": 518,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gpt 4o mini\n",
    "accuracy.rmse(gpt_4o_mini_predictions)\n",
    "accuracy.mae(gpt_4o_mini_predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}