beth_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Imports"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import sys\n",
    "from math import ceil\n",
    "import pandas as pd\n",
    "import dill as pickle\n",
    "# to save results to data directory\n",
    "module_path = '..'\n",
    "if module_path not in sys.path:\n",
    "    sys.path.insert(1, module_path)\n",
    "# increase displayed columns in jupyter notebook\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 300)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import logging\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from src.outlier_model import OutlierModel\n",
    "\n",
    "import matplotlib\n",
    "matplotlib.use(\"pgf\")\n",
    "matplotlib.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "})\n",
    "\n",
    "# increase displayed columns in jupyter notebook\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 300)\n",
    "\n",
    "# temporarily remove deprecation warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')\n",
    "logger = logging.getLogger(\"TimeSeries\")\n",
    "logger.setLevel(logging.INFO)\n",
    "\n",
    "output_dir = 'beth_output'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Dataset"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.legend.Legend at 0x10a5684c0>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_path = '#datasets'\n",
    "df1 = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ip-10-100-1-4.csv',index_col=False)\n",
    "# df = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ip-10-100-1-26.csv',index_col=False) # No evil\n",
    "# df = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ip-10-100-1-95.csv',index_col=False) # No evil\n",
    "df2 = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ip-10-100-1-105.csv',index_col=False) # Some evil\n",
    "# df = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ip-10-100-1-186.csv',index_col=False) # No evil\n",
    "# df = pd.read_csv(f'{base_path}/BETH/labelled_2021may-ubuntu.csv',index_col=False) # No evil\n",
    "frames = [df1, df2]\n",
    "df = pd.concat(frames, ignore_index=True)\n",
    "df.plot(subplots=True, figsize=(16, 16)); plt.legend(loc='best')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "df.head()\n",
    "outlier_key = \"userId\"\n",
    "data_test = df.copy()\n",
    "\n",
    "m = 1000\n",
    "preload_size = 10000\n",
    "std_dev_mult = 3\n",
    "range_mult = 2\n",
    "recent_mult = 2\n",
    "outlier_model = OutlierModel(m=m, std_dev_mult=std_dev_mult, range_mult=range_mult, recent_mult=recent_mult,\n",
    "                             time_series=data_test[:preload_size][outlier_key].astype(np.float64),\n",
    "                             egress=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 11:56:43 AM:  Anomaly at Global index: 11000\n",
      "04/22/2022 11:56:43 AM: max_mp: 13172.8523, metric:7156.672199999999: metric-max_mp: 6016.180100000001 range: 4241.051600000001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 20000\n",
      "Current Global index: 30000\n",
      "Current Global index: 40000\n",
      "Current Global index: 50000\n",
      "Current Global index: 60000\n",
      "Current Global index: 70000\n",
      "Current Global index: 80000\n",
      "Current Global index: 90000\n",
      "Current Global index: 100000\n",
      "Current Global index: 110000\n",
      "Current Global index: 120000\n",
      "Current Global index: 130000\n",
      "Current Global index: 140000\n",
      "Current Global index: 150000\n",
      "Current Global index: 160000\n",
      "Current Global index: 170000\n",
      "Current Global index: 180000\n",
      "Current Global index: 190000\n",
      "Current Global index: 200000\n",
      "Current Global index: 210000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 11:57:58 AM:  Anomaly at Global index: 213090\n",
      "04/22/2022 11:57:58 AM: max_mp: 2716.2331, metric:1071.7787: metric-max_mp: 1644.4543999999999 range: 1708.8236\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 220000\n",
      "Current Global index: 230000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 11:58:07 AM:  Anomaly at Global index: 237204\n",
      "04/22/2022 11:58:07 AM: max_mp: 4256.8639, metric:2466.7595: metric-max_mp: 1790.1044000000002 range: 2442.4585\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 240000\n",
      "Current Global index: 250000\n",
      "Current Global index: 260000\n",
      "Current Global index: 270000\n",
      "Current Global index: 280000\n",
      "Current Global index: 290000\n",
      "Current Global index: 300000\n",
      "Current Global index: 310000\n",
      "Current Global index: 320000\n",
      "Current Global index: 330000\n",
      "Current Global index: 340000\n",
      "Current Global index: 350000\n",
      "Current Global index: 360000\n",
      "Current Global index: 370000\n",
      "Current Global index: 380000\n",
      "Current Global index: 390000\n",
      "Current Global index: 400000\n",
      "Current Global index: 410000\n",
      "Current Global index: 420000\n",
      "Current Global index: 430000\n",
      "Current Global index: 440000\n",
      "Current Global index: 450000\n",
      "Current Global index: 460000\n",
      "Current Global index: 470000\n",
      "Current Global index: 480000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 11:59:42 AM:  Anomaly at Global index: 485354\n",
      "04/22/2022 11:59:42 AM: max_mp: 5386.6102, metric:2327.0199: metric-max_mp: 3059.5903000000003 range: 3386.1303\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 490000\n",
      "Current Global index: 500000\n",
      "Current Global index: 510000\n",
      "Current Global index: 520000\n",
      "Current Global index: 530000\n",
      "Current Global index: 540000\n",
      "Current Global index: 550000\n",
      "Current Global index: 560000\n",
      "Current Global index: 570000\n",
      "Current Global index: 580000\n",
      "Current Global index: 590000\n",
      "Current Global index: 600000\n",
      "Current Global index: 610000\n",
      "Current Global index: 620000\n",
      "Current Global index: 630000\n",
      "Current Global index: 640000\n",
      "Current Global index: 650000\n",
      "Current Global index: 660000\n",
      "Current Global index: 670000\n",
      "Current Global index: 680000\n",
      "Current Global index: 690000\n",
      "Current Global index: 700000\n",
      "Current Global index: 710000\n",
      "Current Global index: 720000\n",
      "Current Global index: 730000\n",
      "Current Global index: 740000\n",
      "Current Global index: 750000\n",
      "Current Global index: 760000\n",
      "Current Global index: 770000\n",
      "Current Global index: 780000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 12:01:36 PM:  Anomaly at Global index: 787618\n",
      "04/22/2022 12:01:36 PM: max_mp: 6206.1965, metric:2062.7696: metric-max_mp: 4143.4269 range: 4714.8508\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 790000\n",
      "Current Global index: 800000\n",
      "Current Global index: 810000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/22/2022 12:01:47 PM:  Anomaly at Global index: 816542\n",
      "04/22/2022 12:01:47 PM: max_mp: 9845.8241, metric:3266.2714: metric-max_mp: 6579.5527 range: 6627.806699999999\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 820000\n",
      "Current Global index: 830000\n",
      "Current Global index: 840000\n",
      "Current Global index: 850000\n",
      "Current Global index: 860000\n",
      "Current Global index: 870000\n",
      "Current Global index: 880000\n",
      "Current Global index: 890000\n"
     ]
    }
   ],
   "source": [
    "fault = False\n",
    "for index, row in data_test[preload_size:].iterrows():\n",
    "    outlier_model.train_one(row[outlier_key])\n",
    "    fault = outlier_model.predict_one(index)\n",
    "    if index % 10000 == 0:\n",
    "        print(f\"Current Global index: {index}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "with open(\"../masters-thesis-graphing/_data/BETH/model.pkl\", 'wb') as handle:\n",
    "    pickle.dump(outlier_model, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open(\"../masters-thesis-graphing/_data/BETH/data.pkl\", 'wb') as handle:\n",
    "    pickle.dump(df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d9d18e9f29cc76176cc64cecda10b08503df27bbeb68d46276fb666f16272d10"
  },
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3 (ipykernel)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}