_scripts/plot_noninteractive_synthetic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from typing import List\n",
    "\n",
    "import pandas\n",
    "\n",
    "def collect_stat(benchmark_result_dir:str, csv_path=None):\n",
    "    if csv_path is None:\n",
    "        csv_path = os.path.join(benchmark_result_dir, \"summary.csv\")\n",
    "        \n",
    "    all_data = []\n",
    "    # Loop through the items in the directory\n",
    "    for item in os.listdir(benchmark_result_dir):\n",
    "        subdir_full_path = os.path.join(benchmark_result_dir, item)\n",
    "        \n",
    "        # Check if the item is a subdirectory\n",
    "        if not (item.startswith('nreqs') and os.path.isdir(subdir_full_path)):\n",
    "            print(f'skipping {item}')\n",
    "            continue\n",
    "        \n",
    "        result_json_file = os.path.join(subdir_full_path, 'benchmark.json')\n",
    "        if not os.path.isfile(result_json_file):\n",
    "            print(f'skipping {item}')\n",
    "            continue\n",
    "        \n",
    "        # Parse dir name to configs \n",
    "        configs = {}\n",
    "        pairs = item.split('.')\n",
    "        for pair in pairs:\n",
    "            key, value = pair.split('_')\n",
    "            try:\n",
    "                value = int(value)\n",
    "            except ValueError:\n",
    "                pass\n",
    "            # Add the key-value pair to the dictionary\n",
    "            configs[key] = value\n",
    "            \n",
    "        # Read the benchmark result\n",
    "        with open(result_json_file) as file:\n",
    "            metrics = json.load(file)\n",
    "            \n",
    "        all_data.append({**configs, **metrics})\n",
    "        \n",
    "    df = pandas.DataFrame(all_data)\n",
    "    df.to_csv(csv_path, index=False)\n",
    "    \n",
    "    return csv_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from typing import Iterable, Union, List, Dict\n",
    "\n",
    "def plot_multi_barchart(group_ids:Union[List[int], List[str]],\n",
    "                        data_dict:Dict[str, Iterable], ax:plt.Axes=None,\n",
    "                        put_text_on_bars:bool=False):\n",
    "    ngrps = len(group_ids)\n",
    "    nbars_per_grp = len(data_dict)\n",
    "    bar_width = 1. / (nbars_per_grp + 1)\n",
    "    opacity = 0.8\n",
    "    # opacity = 1.0\n",
    "\n",
    "    leftmost = np.arange(ngrps) - (nbars_per_grp*bar_width / 2)\n",
    "\n",
    "    if ax is None:\n",
    "        _, ax = plt.subplots()\n",
    "\n",
    "    for idx, (name, data) in enumerate(data_dict.items()):\n",
    "        assert len(data) == ngrps\n",
    "        center_offset_to_leftmost = bar_width / 2 + bar_width*idx\n",
    "        bars = plt.bar(\n",
    "            leftmost + center_offset_to_leftmost , # center of the bars\n",
    "            data, # heights of the bars\n",
    "            bar_width,\n",
    "            alpha=opacity,\n",
    "            label=name\n",
    "        )\n",
    "        if put_text_on_bars:\n",
    "            for i in range(ngrps):\n",
    "                # print(data[i])\n",
    "                plt.text(leftmost[i]+center_offset_to_leftmost, data[i], f'{data[i]:.1f}', ha='center', size='x-small')\n",
    "    \n",
    "    ax.set_xticks(np.arange(ngrps), group_ids)\n",
    "    \n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def plot_figure(csv_path:str,\n",
    "                workload=(128, 256),\n",
    "                metric_plot='generate_tok_per_sec',\n",
    "                y_lim=None,\n",
    "                ftype='line',\n",
    "                save_name=None):\n",
    "    assert ftype in {'line', 'bar'}\n",
    "    # metric_plot = 'throughput'\n",
    "    line_styles = ['-o', '-D', '->']\n",
    "    df = pd.read_csv(csv_path)\n",
    "    condition = (df['ctxlen']==workload[0]) & (df['outlen']==workload[1])\n",
    "    condition_dict = { \n",
    "        'vLLM-RA (ours)': condition & (df['backend']=='vllm+'),\n",
    "        'vLLM+': condition & (df['backend']=='vllm+pc'), \n",
    "        'vLLM': condition & (df['backend']=='vllm')\n",
    "    }\n",
    "    \n",
    "    data_dict = {}\n",
    "    for name, cond in condition_dict.items():\n",
    "        # print(df)\n",
    "        # print(df['reqrate'])\n",
    "        view = df[cond].sort_values('prefixlen')\n",
    "        x = view['prefixlen'].to_numpy(dtype=np.int32)\n",
    "        y = view[metric_plot].to_numpy(dtype=np.float32)\n",
    "        data_dict[name] = (x, y)\n",
    "\n",
    "    # try:\n",
    "    #     plt_context = plt.style.context(['light', 'grid'])\n",
    "    #     with plt_context:\n",
    "    #         pass\n",
    "    # except:\n",
    "    #     import contextlib\n",
    "    #     plt_context = contextlib.nullcontext()\n",
    "    \n",
    "    plt_context = plt.style.context(['grid'])\n",
    "\n",
    "    with plt_context:\n",
    "        figure, ax = plt.subplots()\n",
    "        if ftype == 'line':\n",
    "            for idx, (label, data) in enumerate(data_dict.items()):\n",
    "                x, y = data\n",
    "                ax.plot(x, y, line_styles[idx], label=label)\n",
    "            if y_lim:\n",
    "                ax.set_ylim(y_lim)\n",
    "            ax.set_xlabel('System prompt length (#toks)')\n",
    "            # ax.set_ylabel('Throughput (tok/s)')\n",
    "        else:\n",
    "            bar_dict = {k:v for k, (_, v) in data_dict.items()}\n",
    "            group_ids = list(data_dict.values())[0][0]\n",
    "            plot_multi_barchart(group_ids=group_ids, data_dict=bar_dict, ax=ax)\n",
    "        \n",
    "        # ax.set_ylabel('Throughput (tok/s)')\n",
    "        # ax.set_xlabel('System Prompt Length (#tokens)')\n",
    "        # ax.set_title(f'user prompt length={workload[0]}, output length={workload[1]}')\n",
    "        # ax.legend()\n",
    "    plt.tight_layout()\n",
    "    save_dir = os.path.dirname(csv_path)\n",
    "    if save_name:\n",
    "        save_path = os.path.join(save_dir, save_name)\n",
    "        plt.savefig(save_path)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bench='noninteractive_bench_synthetic'\n",
    "exp_root = f'../outputs/{bench}'\n",
    "\n",
    "csv_root = f'../stat/{bench}'\n",
    "os.makedirs(csv_root, exist_ok=True)\n",
    "\n",
    "collect = False\n",
    "\n",
    "figsize=(8, 6)\n",
    "plt.rcParams.update({'font.size': 16,\n",
    "                     'font.weight': 'bold',\n",
    "                     'lines.markersize': 10,\n",
    "                     'lines.linewidth': 2,\n",
    "                     'font.family': 'serif',\n",
    "                     'font.serif': 'Times New Roman',\n",
    "                     'axes.linewidth': 2\n",
    "                     })\n",
    "\n",
    "if collect:\n",
    "    for gpu in os.listdir(exp_root):\n",
    "        gpu_dir = os.path.join(exp_root, gpu)\n",
    "        for model in os.listdir(gpu_dir):\n",
    "            benchmark_result_dir = os.path.join(gpu_dir, model)\n",
    "            csv_path = os.path.join(csv_root, f'{gpu}.{model}.csv')\n",
    "            collect_stat(benchmark_result_dir, csv_path=csv_path)\n",
    "\n",
    "for file in os.listdir(csv_root):\n",
    "    if file.endswith('.csv'):\n",
    "        gpu, model, _ = file.split('.')\n",
    "        csv_path = os.path.join(csv_root,file)\n",
    "        if (gpu == 'NVIDIA-A40') and (model == 'Llama-2-7b-hf'):\n",
    "            for workload in [ (64, 128), (128, 256), (256, 512) ]:\n",
    "                print(gpu, model, workload)\n",
    "                save_name = f'syn.{workload}.{gpu}.{model}.svg'\n",
    "                plot_figure(csv_path, workload=workload, ftype='line', save_name=save_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "plot",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}