notebooks/00_quick_start/sarplus_movielens.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running SAR on MovieLens (sarplus)\n",
    "\n",
    "This notebook provides an example of how to utilize and evaluate SAR using sarplus, which is based on PySpark (SQL + C++).\n",
    "\n",
    "The biggest advantage of sarplus is scalability of the prediction part, as it's the only limitation is that the similiarity matrix needs to fit in memory on each worker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "System version: 3.6.2 |Anaconda, Inc.| (default, Sep 21 2017, 18:29:43) \n",
      "[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "Pandas version: 0.20.3\n"
     ]
    }
   ],
   "source": [
    "# set the environment path to find Recommenders\n",
    "import sys\n",
    "sys.path.append(\"../../\")\n",
    "\n",
    "from reco_utils.dataset.url_utils import maybe_download\n",
    "from reco_utils.dataset.python_splitters import python_random_split\n",
    "import reco_utils.evaluation.python_evaluation as evaluation\n",
    "\n",
    "import itertools\n",
    "import os\n",
    "import pandas as pd\n",
    "from pyspark.sql import SparkSession\n",
    "from pysarplus import SARPlus\n",
    "\n",
    "print(\"System version: {}\".format(sys.version))\n",
    "print(\"Pandas version: {}\".format(pd.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Download the MovieLens dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>MovieId</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>196</td>\n",
       "      <td>242</td>\n",
       "      <td>3</td>\n",
       "      <td>881250949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>186</td>\n",
       "      <td>302</td>\n",
       "      <td>3</td>\n",
       "      <td>891717742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22</td>\n",
       "      <td>377</td>\n",
       "      <td>1</td>\n",
       "      <td>878887116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>51</td>\n",
       "      <td>2</td>\n",
       "      <td>880606923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>166</td>\n",
       "      <td>346</td>\n",
       "      <td>1</td>\n",
       "      <td>886397596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserId  MovieId  Rating  Timestamp\n",
       "0     196      242       3  881250949\n",
       "1     186      302       3  891717742\n",
       "2      22      377       1  878887116\n",
       "3     244       51       2  880606923\n",
       "4     166      346       1  886397596"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filepath = maybe_download(\"http://files.grouplens.org/datasets/movielens/ml-100k/u.data\", \"ml-100k.data\")\n",
    "data = pd.read_csv(filepath, sep=\"\\t\", names=[\"UserId\", \"MovieId\", \"Rating\", \"Timestamp\"])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Split the data using the python random splitter provided in utilities:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train, test = python_random_split(data)\n",
    "header = {\n",
    "        \"col_user\": \"UserId\",\n",
    "        \"col_item\": \"MovieId\",\n",
    "        \"col_rating\": \"Rating\",\n",
    "        \"col_timestamp\": \"Timestamp\",\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Start Spark and load sar+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SUBMIT_ARGS = \"--packages eisber:sarplus:0.2.2 pyspark-shell\"\n",
    "os.environ[\"PYSPARK_SUBMIT_ARGS\"] = SUBMIT_ARGS\n",
    "\n",
    "spark = (\n",
    "    SparkSession.builder.appName(\"sample\")\n",
    "    .master(\"local[*]\")\n",
    "    .config(\"memory\", \"4G\")\n",
    "    .config(\"spark.sql.shuffle.partitions\", \"10\")\n",
    "    .config(\"spark.sql.crossJoin.enabled\", True)\n",
    "    .config(\"spark.ui.enabled\", False)\n",
    "    .getOrCreate()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the SAR model on our training data, and get the top-k recommendations for our testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sarplus:sarplus.fit 1/2: compute item cooccurences...\n",
      "INFO:sarplus:sarplus.fit 2/2: compute similiarity metric jaccard...\n",
      "INFO:sarplus:sarplus.recommend_k_items 1/3: create item index\n",
      "INFO:sarplus:sarplus.recommend_k_items 2/3: prepare similarity matrix\n",
      "INFO:sarplus:sarplus.recommend_k_items 3/3: compute recommendations\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>MovieId</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7960</th>\n",
       "      <td>796</td>\n",
       "      <td>191</td>\n",
       "      <td>141.250412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7961</th>\n",
       "      <td>796</td>\n",
       "      <td>208</td>\n",
       "      <td>144.016556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7962</th>\n",
       "      <td>796</td>\n",
       "      <td>99</td>\n",
       "      <td>145.766876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7963</th>\n",
       "      <td>796</td>\n",
       "      <td>28</td>\n",
       "      <td>147.205063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7964</th>\n",
       "      <td>796</td>\n",
       "      <td>265</td>\n",
       "      <td>148.355103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7965</th>\n",
       "      <td>796</td>\n",
       "      <td>79</td>\n",
       "      <td>152.824890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7966</th>\n",
       "      <td>796</td>\n",
       "      <td>566</td>\n",
       "      <td>153.102249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7967</th>\n",
       "      <td>796</td>\n",
       "      <td>174</td>\n",
       "      <td>154.093216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7968</th>\n",
       "      <td>796</td>\n",
       "      <td>216</td>\n",
       "      <td>154.755188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7969</th>\n",
       "      <td>796</td>\n",
       "      <td>234</td>\n",
       "      <td>155.419418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      UserId  MovieId       score\n",
       "7960     796      191  141.250412\n",
       "7961     796      208  144.016556\n",
       "7962     796       99  145.766876\n",
       "7963     796       28  147.205063\n",
       "7964     796      265  148.355103\n",
       "7965     796       79  152.824890\n",
       "7966     796      566  153.102249\n",
       "7967     796      174  154.093216\n",
       "7968     796      216  154.755188\n",
       "7969     796      234  155.419418"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = spark.createDataFrame(train)\n",
    "df_test = spark.createDataFrame(test)\n",
    "\n",
    "model = SARPlus(spark, **header)\n",
    "model.fit(df_train, similarity_type='jaccard', \n",
    "          time_decay_coefficient=30, time_now=None, timedecay_formula=True)\n",
    "\n",
    "top_k = model.recommend_k_items(df_test, 'sarplus_cache', top_k=10, remove_seen=True)\\\n",
    "    .toPandas()\n",
    "\n",
    "top_k[top_k.UserId == 796].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Evaluate how well SAR performs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metrics</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Top K</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MAP</td>\n",
       "      <td>0.105815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NDCG</td>\n",
       "      <td>0.373197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Precision@K</td>\n",
       "      <td>0.326617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Recall@K</td>\n",
       "      <td>0.175957</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       metrics      value\n",
       "0        Top K  10.000000\n",
       "1          MAP   0.105815\n",
       "2         NDCG   0.373197\n",
       "3  Precision@K   0.326617\n",
       "4     Recall@K   0.175957"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header_eval = {\n",
    "    \"col_user\":\"UserId\", \n",
    "    \"col_item\":\"MovieId\", \n",
    "    \"col_rating\":\"Rating\", \n",
    "    \"col_prediction\":\"score\"\n",
    "}\n",
    "\n",
    "pd.DataFrame({\n",
    "    'metrics': ['Top K', 'MAP', 'NDCG', 'Precision@K', 'Recall@K'],\n",
    "    'value': [\n",
    "        10,\n",
    "        evaluation.map_at_k(test, top_k, **header_eval),\n",
    "        evaluation.ndcg_at_k(test, top_k, **header_eval),\n",
    "        evaluation.precision_at_k(test, top_k, **header_eval),\n",
    "        evaluation.recall_at_k(test, top_k, **header_eval)\n",
    "    ]\n",
    "})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (recommender)",
   "language": "python",
   "name": "recommender"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}