From 5b1e55189e2aa44e984537d7bded3f5bb74e74d1 Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 16 Apr 2020 16:11:25 +0000 Subject: [PATCH 01/17] update nni to 1.5 --- notebooks/02_model/ncf_deep_dive.ipynb | 236 ++++- .../nni_ncf.ipynb | 994 ++++++++++++++++++ scripts/generate_conda_file.py | 4 +- 3 files changed, 1187 insertions(+), 47 deletions(-) create mode 100644 notebooks/04_model_select_and_optimize/nni_ncf.ipynb diff --git a/notebooks/02_model/ncf_deep_dive.ipynb b/notebooks/02_model/ncf_deep_dive.ipynb index 03ff402802..6d2a7a7a72 100644 --- a/notebooks/02_model/ncf_deep_dive.ipynb +++ b/notebooks/02_model/ncf_deep_dive.ipynb @@ -30,13 +30,31 @@ "execution_count": 1, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", + "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "System version: 3.6.8 |Anaconda, Inc.| (default, Dec 30 2018, 01:22:34) \n", + "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", - "Pandas version: 0.24.2\n", + "Pandas version: 0.25.3\n", "Tensorflow version: 1.12.0\n" ] } @@ -206,7 +224,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "4.93MB [00:00, 18.4MB/s] \n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.1kKB/s]\n" ] }, { @@ -380,7 +398,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 662.0558905601501 seconds for training.\n" + "Took 410.15181515900076 seconds for training.\n" ] } ], @@ -443,31 +461,31 @@ " 0\n", " 1.0\n", " 149.0\n", - " 0.150561\n", + " 0.017463\n", " \n", " \n", " 1\n", " 1.0\n", " 88.0\n", - " 0.307006\n", + " 0.407677\n", " \n", " \n", " 2\n", " 1.0\n", " 101.0\n", - " 0.346554\n", + " 0.375374\n", " \n", " \n", " 3\n", " 1.0\n", " 110.0\n", - " 0.185647\n", + " 0.026372\n", " \n", " \n", " 4\n", " 1.0\n", " 103.0\n", - " 0.002877\n", + " 0.020589\n", " \n", " \n", "\n", @@ -475,11 +493,11 @@ ], "text/plain": [ " userID itemID prediction\n", - "0 1.0 149.0 0.150561\n", - "1 1.0 88.0 0.307006\n", - "2 1.0 101.0 0.346554\n", - "3 1.0 110.0 0.185647\n", - "4 1.0 103.0 0.002877" + "0 1.0 149.0 0.017463\n", + "1 1.0 88.0 0.407677\n", + "2 1.0 101.0 0.375374\n", + "3 1.0 110.0 0.026372\n", + "4 1.0 103.0 0.020589" ] }, "execution_count": 8, @@ -514,7 +532,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 2.801203966140747 seconds for prediction.\n" + "Took 2.4053909909998765 seconds for prediction.\n" ] } ], @@ -546,10 +564,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.044980\n", - "NDCG:\t0.189388\n", - "Precision@K:\t0.170626\n", - "Recall@K:\t0.094332\n" + "MAP:\t0.047199\n", + "NDCG:\t0.193792\n", + "Precision@K:\t0.175504\n", + "Recall@K:\t0.097036\n" ] } ], @@ -592,8 +610,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "HR:\t0.489403\n", - "NDCG:\t0.385408\n" + "HR:\t0.488684\n", + "NDCG:\t0.383331\n" ] } ], @@ -674,7 +692,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 514.4518468379974 seconds for training.\n" + "Took 332.92922498900043 seconds for training.\n" ] } ], @@ -717,7 +735,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 565.3551139831543 seconds for training.\n" + "Took 367.1780747690009 seconds for training.\n" ] } ], @@ -742,16 +760,7 @@ "cell_type": "code", "execution_count": 16, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from .pretrain/GMF/model.ckpt\n", - "INFO:tensorflow:Restoring parameters from .pretrain/MLP/model.ckpt\n" - ] - } - ], + "outputs": [], "source": [ "model = NCF (\n", " n_users=data.n_users, \n", @@ -778,7 +787,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 660.8611929416656 seconds for training.\n" + "Took 414.2128918319995 seconds for training.\n" ] } ], @@ -807,7 +816,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 2.8479833602905273 seconds for prediction.\n" + "Took 2.5282814109996252 seconds for prediction.\n" ] } ], @@ -839,10 +848,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.041600\n", - "NDCG:\t0.174057\n", - "Precision@K:\t0.160657\n", - "Recall@K:\t0.091654\n" + "MAP:\t0.043843\n", + "NDCG:\t0.182116\n", + "Precision@K:\t0.166702\n", + "Recall@K:\t0.093253\n" ] } ], @@ -860,9 +869,146 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " \n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "map": 0.04719861331116455 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:3: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " This is separate from the ipykernel package so we can avoid doing imports until\n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "ndcg": 0.3833305241576571 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " after removing the cwd from sys.path.\n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "precision": 0.17550371155885472 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " \"\"\"\n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "recall": 0.09703642848424668 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " \n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "map2": 0.04384271262964714 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " import sys\n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "ndcg2": 0.18211610675334622 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:8: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " \n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "precision2": 0.16670201484623542 + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + " if __name__ == '__main__':\n" + ] + }, + { + "data": { + "application/papermill.record+json": { + "recall2": 0.09325259530179235 + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Record results with papermill for tests\n", "pm.record(\"map\", eval_map)\n", @@ -919,9 +1065,9 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "reco_gpu", + "display_name": "Python (recommenders)", "language": "python", - "name": "reco_gpu" + "name": "recommenders" }, "language_info": { "codemirror_mode": { @@ -933,9 +1079,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.10" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb new file mode 100644 index 0000000000..72afc94679 --- /dev/null +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -0,0 +1,994 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.
\n", + "Licensed under the MIT License.\n", + "
\n", + "# Hyperparameter Tuning for Matrix Factorization Using the Neural Network Intelligence Toolkit" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters of a matrix factorization model. In particular, we optimize the hyperparameters of [Surprise SVD](https://surprise.readthedocs.io/en/stable/matrix_factorization.html).\n", + "\n", + "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. NNI comes with [several tuning algorithms](https://nni.readthedocs.io/en/latest/Builtin_Tuner.html) built in. It also allows users to [define their own general purpose tuners](https://nni.readthedocs.io/en/latest/Customize_Tuner.html). NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", + "\n", + "In this notebook we execute several NNI _experiments_ on the same data sets obtained from Movielens with a training-validation-test split. Each experiment corresponds to one of the built-in tuning algorithms. It consists of many parallel _trials_, each of which corresponds to a choice of hyperparameters sampled by the tuning algorithm. All the experiments require a call to the same [python script](../../reco_utils/nni/svd_training.py) for training the SVD model and evaluating rating and ranking metrics on the test data. This script has been adapted from the [Surprise SVD notebook](../02_model/surprise_svd_deep_dive.ipynb) with only a few changes. In all experiments, we maximize precision@10. \n", + "\n", + "For this notebook we use a _local machine_ as the training platform (this can be any machine running the `reco_base` conda environment). In this case, NNI uses the available processors of the machine to parallelize the trials, subject to the value of `trialConcurrency` we specify in the configuration. Our runs and the results we report were obtained on a [Standard_D16_v3 virtual machine](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/sizes-general#dv3-series-1) with 16 vcpus and 64 GB memory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Global Settings" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", + "[GCC 7.3.0]\n", + "Surprise version: 1.1.0\n", + "NNI version: 0.5.2.1.1\n" + ] + } + ], + "source": [ + "import sys\n", + "sys.path.append(\"../../\")\n", + "import json\n", + "import os\n", + "import surprise\n", + "import papermill as pm\n", + "import pandas as pd\n", + "import shutil\n", + "import subprocess\n", + "import yaml\n", + "import pkg_resources\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "import reco_utils\n", + "from reco_utils.common.timer import Timer\n", + "from reco_utils.dataset import movielens\n", + "from reco_utils.dataset.python_splitters import python_random_split\n", + "from reco_utils.evaluation.python_evaluation import rmse, precision_at_k, ndcg_at_k\n", + "from reco_utils.tuning.nni.nni_utils import (check_experiment_status, check_stopped, check_metrics_written, get_trials,\n", + " stop_nni, start_nni)\n", + "from reco_utils.recommender.surprise.surprise_utils import predict, compute_ranking_predictions\n", + "\n", + "print(\"System version: {}\".format(sys.version))\n", + "print(\"Surprise version: {}\".format(surprise.__version__))\n", + "print(\"NNI version: {}\".format(pkg_resources.get_distribution(\"nni\").version))\n", + "\n", + "tmp_dir = TemporaryDirectory()\n", + "\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Prepare Dataset\n", + "1. Download data and split into training, validation and test sets\n", + "2. Store the data sets to a local directory." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters used by papermill\n", + "# Select Movielens data size: 100k, 1m\n", + "MOVIELENS_DATA_SIZE = '100k'\n", + "SURPRISE_READER = 'ml-100k'\n", + "TMP_DIR = tmp_dir.name\n", + "NUM_EPOCHS = 30\n", + "MAX_TRIAL_NUM = 100\n", + "# time (in seconds) to wait for each tuning experiment to complete\n", + "WAITING_TIME = 20\n", + "MAX_RETRIES = 400 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.5kKB/s]\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIDitemIDrating
01962423.0
11863023.0
2223771.0
3244512.0
41663461.0
\n", + "
" + ], + "text/plain": [ + " userID itemID rating\n", + "0 196 242 3.0\n", + "1 186 302 3.0\n", + "2 22 377 1.0\n", + "3 244 51 2.0\n", + "4 166 346 1.0" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = movielens.load_pandas_df(\n", + " size=MOVIELENS_DATA_SIZE,\n", + " header=[\"userID\", \"itemID\", \"rating\"]\n", + ")\n", + "\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train, validation, test = python_random_split(data, [0.7, 0.15, 0.15])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "LOG_DIR = os.path.join(TMP_DIR, \"experiments\")\n", + "os.makedirs(LOG_DIR, exist_ok=True)\n", + "\n", + "DATA_DIR = os.path.join(TMP_DIR, \"data\") \n", + "os.makedirs(DATA_DIR, exist_ok=True)\n", + "\n", + "TRAIN_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_train.pkl\"\n", + "train.to_pickle(os.path.join(DATA_DIR, TRAIN_FILE_NAME))\n", + "\n", + "VAL_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_val.pkl\"\n", + "validation.to_pickle(os.path.join(DATA_DIR, VAL_FILE_NAME))\n", + "\n", + "TEST_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_test.pkl\"\n", + "test.to_pickle(os.path.join(DATA_DIR, TEST_FILE_NAME))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Prepare Hyperparameter Tuning " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We now prepare a training script [svd_training_nni.py](../../reco_utils/nni/svd_training.py) for the hyperparameter tuning, which will log our target metrics such as precision, NDCG, RMSE.\n", + "We define the arguments of the script and the search space for the hyperparameters. All the parameter values will be passed to our training script.
\n", + "Note that we specify _precision@10_ as the primary metric. We will also instruct NNI (in the configuration file) to _maximize_ the primary metric. This is passed as an argument in the training script and the evaluated metric is returned through the NNI python library. In addition, we also evaluate RMSE and NDCG@10. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `script_params` below are the parameters of the training script that are fixed (unlike `hyper_params` which are tuned). In particular, `VERBOSE, BIASED, RANDOM_STATE, NUM_EPOCHS` are parameters used in the [SVD method](../02_model/surprise_svd_deep_dive.ipynb) and `REMOVE_SEEN` removes the training data from the recommended items. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "EXP_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_svd_model\"\n", + "PRIMARY_METRIC = \"precision_at_k\"\n", + "RATING_METRICS = [\"rmse\"]\n", + "RANKING_METRICS = [\"precision_at_k\", \"ndcg_at_k\"] \n", + "USERCOL = \"userID\"\n", + "ITEMCOL = \"itemID\"\n", + "REMOVE_SEEN = True\n", + "K = 10\n", + "RANDOM_STATE = 42\n", + "VERBOSE = True\n", + "BIASED = True\n", + "\n", + "script_params = \" \".join([\n", + " \"--datastore\", DATA_DIR,\n", + " \"--train-datapath\", TRAIN_FILE_NAME,\n", + " \"--validation-datapath\", VAL_FILE_NAME,\n", + " \"--surprise-reader\", SURPRISE_READER,\n", + " \"--rating-metrics\", \" \".join(RATING_METRICS),\n", + " \"--ranking-metrics\", \" \".join(RANKING_METRICS),\n", + " \"--usercol\", USERCOL,\n", + " \"--itemcol\", ITEMCOL,\n", + " \"--k\", str(K),\n", + " \"--random-state\", str(RANDOM_STATE),\n", + " \"--epochs\", str(NUM_EPOCHS),\n", + " \"--primary-metric\", PRIMARY_METRIC\n", + "])\n", + "\n", + "if BIASED:\n", + " script_params += \" --biased\"\n", + "if VERBOSE:\n", + " script_params += \" --verbose\"\n", + "if REMOVE_SEEN:\n", + " script_params += \" --remove-seen\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# hyperparameters search space\n", + "# We do not set 'lr_all' and 'reg_all' because they will be overriden by the other lr_ and reg_ parameters\n", + "\n", + "hyper_params = {\n", + " 'n_factors': {\"_type\": \"choice\", \"_value\": [10, 50, 100, 150, 200]},\n", + " 'init_mean': {\"_type\": \"uniform\", \"_value\": [-0.5, 0.5]},\n", + " 'init_std_dev': {\"_type\": \"uniform\", \"_value\": [0.01, 0.2]},\n", + " 'lr_bu': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", + " 'lr_bi': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", + " 'lr_pu': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", + " 'lr_qi': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", + " 'reg_bu': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]},\n", + " 'reg_bi': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}, \n", + " 'reg_pu': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}, \n", + " 'reg_qi': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "with open(os.path.join(TMP_DIR, 'search_space_svd.json'), 'w') as fp:\n", + " json.dump(hyper_params, fp)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also create a yaml file for the configuration of the trials and the tuning algorithm to be used (in this experiment we use the [TPE tuner](https://nni.readthedocs.io/en/latest/hyperoptTuner.html)). " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "config = {\n", + " \"authorName\": \"default\",\n", + " \"experimentName\": \"surprise_svd\",\n", + " \"trialConcurrency\": 8,\n", + " \"maxExecDuration\": \"1h\",\n", + " \"maxTrialNum\": MAX_TRIAL_NUM,\n", + " \"trainingServicePlatform\": \"local\",\n", + " # The path to Search Space\n", + " \"searchSpacePath\": \"search_space_svd.json\",\n", + " \"useAnnotation\": False,\n", + " \"logDir\": LOG_DIR,\n", + " \"tuner\": {\n", + " \"builtinTunerName\": \"TPE\",\n", + " \"classArgs\": {\n", + " #choice: maximize, minimize\n", + " \"optimize_mode\": \"maximize\"\n", + " }\n", + " },\n", + " # The path and the running command of trial\n", + " \"trial\": {\n", + " \"command\": sys.prefix + \"/bin/python svd_training.py\" + \" \" + script_params,\n", + " \"codeDir\": os.path.join(os.path.split(os.path.abspath(reco_utils.__file__))[0], \"tuning\", \"nni\"),\n", + " \"gpuNum\": 0\n", + " }\n", + "}\n", + " \n", + "with open(os.path.join(TMP_DIR, \"config_svd.yml\"), \"w\") as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Execute NNI Trials\n", + "\n", + "The conda environment comes with NNI installed, which includes the command line tool `nnictl` for controlling and getting information about NNI experiments.
\n", + "To start the NNI tuning trials from the command line, execute the following command:
\n", + "`nnictl create --config `
\n", + "In the cell below, we call this command programmatically.
\n", + "You can see the progress of the experiment by using the URL links output by the above command.\n", + "\n", + "![](https://recodatasets.blob.core.windows.net/images/nn1.png)\n", + "\n", + "![](https://recodatasets.blob.core.windows.net/images/nn2.png)\n", + "\n", + "![](https://recodatasets.blob.core.windows.net/images/nn3.png)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Make sure that there is no experiment running\n", + "stop_nni()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "config_path = os.path.join(TMP_DIR, 'config_svd.yml')\n", + "nni_env = os.environ.copy()\n", + "nni_env['PATH'] = sys.prefix + '/bin:' + nni_env['PATH']\n", + "proc = subprocess.run([sys.prefix + '/bin/nnictl', 'create', '--config', config_path], env=nni_env)\n", + "if proc.returncode != 0:\n", + " raise RuntimeError(\"'nnictl create' failed with code %d\" % proc.returncode)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "with Timer() as time_tpe:\n", + " check_experiment_status(wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5. Show Results\n", + "\n", + "The trial with the best metric and the corresponding metrics and hyperparameters can also be read from the Web UI\n", + "\n", + "![](https://recodatasets.blob.core.windows.net/images/nni4.png)\n", + "\n", + "or from the JSON file created by the training script. Below, we do this programmatically using [nni_utils.py](../../reco_utils/nni/nni_utils.py) " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "trials, best_metrics, best_params, best_trial_path = get_trials('maximize')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 1.0148825026253758,\n", + " 'ndcg_at_k': 0.07073059758480135,\n", + " 'precision_at_k': 0.05867944621938233}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "best_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'parameter_id': 20,\n", + " 'parameter_source': 'algorithm',\n", + " 'parameters': {'n_factors': 10,\n", + " 'init_mean': -0.20640543895286745,\n", + " 'init_std_dev': 0.041877648647603426,\n", + " 'lr_bu': 0.010258651938050185,\n", + " 'lr_bi': 8.286686919368394e-05,\n", + " 'lr_pu': 0.09981032112256295,\n", + " 'lr_qi': 0.06900431839934824,\n", + " 'reg_bu': 0.012030372162218295,\n", + " 'reg_bi': 0.10285996563173667,\n", + " 'reg_pu': 0.21894910939861875,\n", + " 'reg_qi': 0.8864729891838474}}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "best_params" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/tmp/tmp1542zrho/experiments/oV4pQDZF/trials/uxVdI'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "best_trial_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This directory path is where info about the trial can be found, including logs, parameters and the model that was learned. To evaluate the metrics on the test data, we get the SVD model that was saved as `model.dump` in the training script." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "svd = surprise.dump.load(os.path.join(best_trial_path, \"model.dump\"))[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following function computes all the metrics given an SVD model." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def compute_test_results(svd):\n", + " test_results = {}\n", + " predictions = predict(svd, test, usercol=\"userID\", itemcol=\"itemID\")\n", + " for metric in RATING_METRICS:\n", + " test_results[metric] = eval(metric)(test, predictions)\n", + "\n", + " all_predictions = compute_ranking_predictions(svd, train, usercol=\"userID\", itemcol=\"itemID\", remove_seen=REMOVE_SEEN)\n", + " for metric in RANKING_METRICS:\n", + " test_results[metric] = eval(metric)(test, all_predictions, col_prediction='prediction', k=K)\n", + " return test_results" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'rmse': 1.0119318175407621, 'precision_at_k': 0.09872068230277187, 'ndcg_at_k': 0.11604975035480133}\n" + ] + } + ], + "source": [ + "test_results_tpe = compute_test_results(svd)\n", + "print(test_results_tpe)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. More Tuning Algorithms\n", + "We now apply other tuning algorithms supported by NNI to the same problem. For details about these tuners, see the [NNI docs.](https://nni.readthedocs.io/en/latest/tuners.html#)\n", + "The only change needed is in the relevant entry in the configuration file." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In summary, the tuners used in this notebook are the following:\n", + "- Tree-structured Parzen Estimator (TPE), within the Sequential Model-Based Optimization (SMBO) framework,\n", + "- SMAC, also an instance of SMBO,\n", + "- Hyperband\n", + "- Metis, an implementation of Bayesian optimization with Gaussian Processes\n", + "- a Naive Evolutionary algorithm\n", + "- an Annealing method for sampling, and \n", + "- plain Random Search as a baseline. \n", + "\n", + "For more details and references to the relevant literature, see the [NNI github](https://github.com/Microsoft/nni/blob/master/docs/en_US/Builtin_Tuner.md)." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/tmp/tmp1542zrho/config_svd.yml\n" + ] + } + ], + "source": [ + "# Random search\n", + "config['tuner']['builtinTunerName'] = 'Random'\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))\n", + " print(config_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "with Timer() as time_random:\n", + " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "test_results_random = compute_test_results(svd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Annealing\n", + "config['tuner']['builtinTunerName'] = 'Anneal'\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "with Timer() as time_anneal:\n", + " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "test_results_anneal = compute_test_results(svd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Naive evolutionary search\n", + "config['tuner']['builtinTunerName'] = 'Evolution'\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "with Timer() as time_evolution:\n", + " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "test_results_evolution = compute_test_results(svd)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The SMAC tuner requires to have been installed with the following command
\n", + "`nnictl package install --name=SMAC`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SMAC\n", + "config['tuner']['builtinTunerName'] = 'SMAC'\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check if installed\n", + "proc = subprocess.run([sys.prefix + '/bin/nnictl', 'package', 'show'], stdout=subprocess.PIPE)\n", + "if proc.returncode != 0:\n", + " raise RuntimeError(\"'nnictl package show' failed with code %d\" % proc.returncode)\n", + "if 'SMAC' not in proc.stdout.decode().strip().split():\n", + " proc = subprocess.run([sys.prefix + '/bin/nnictl', 'package', 'install', '--name=SMAC'])\n", + " if proc.returncode != 0:\n", + " raise RuntimeError(\"'nnictl package install' failed with code %d\" % proc.returncode)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Skipping SMAC optimization for now\n", + "# stop_nni()\n", + "with Timer() as time_smac:\n", + "# start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + " pass\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#check_metrics_written()\n", + "#svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "#test_results_smac = compute_test_results(svd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Metis\n", + "config['tuner']['builtinTunerName'] = 'MetisTuner'\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "with Timer() as time_metis:\n", + " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written()\n", + "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "test_results_metis = compute_test_results(svd)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hyperband follows a different style of configuration from other tuners. See [the NNI documentation](https://nni.readthedocs.io/en/latest/hyperbandAdvisor.html). Note that the [training script](../../reco_utils/nni/svd_training.py) needs to be adjusted as well, since each Hyperband trial receives an additional parameter `STEPS`, which corresponds to the resource allocation _ri_ in the [Hyperband algorithm](https://arxiv.org/pdf/1603.06560.pdf). In this example, we used `STEPS` in combination with `R` to determine the number of epochs that SVD will run for in every trial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Hyperband\n", + "config['advisor'] = {\n", + " 'builtinAdvisorName': 'Hyperband',\n", + " 'classArgs': {\n", + " 'R': NUM_EPOCHS,\n", + " 'eta': 3,\n", + " 'optimize_mode': 'maximize'\n", + " }\n", + "}\n", + "config.pop('tuner')\n", + "with open(config_path, 'w') as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "with Timer() as time_hyperband:\n", + " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check_metrics_written()\n", + "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", + "test_results_hyperband = compute_test_results(svd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_results_tpe.update({'time': time_tpe.interval})\n", + "test_results_random.update({'time': time_random.interval})\n", + "test_results_anneal.update({'time': time_anneal.interval})\n", + "test_results_evolution.update({'time': time_evolution.interval})\n", + "#test_results_smac.update({'time': time_smac.interval})\n", + "test_results_metis.update({'time': time_metis.interval})\n", + "test_results_hyperband.update({'time': time_hyperband.interval})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "algos = [\"TPE\", \n", + " \"Random Search\", \n", + " \"Annealing\", \n", + " \"Evolution\", \n", + " #\"SMAC\", \n", + " \"Metis\", \n", + " \"Hyperband\"]\n", + "res_df = pd.DataFrame(index=algos,\n", + " data=[res for res in [test_results_tpe, \n", + " test_results_random, \n", + " test_results_anneal, \n", + " test_results_evolution, \n", + " #test_results_smac, \n", + " test_results_metis, \n", + " test_results_hyperband]] \n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "res_df.sort_values(by=\"precision_at_k\", ascending=False).round(3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we see in the table above, _annealing_ performs best with respect to the primary metric (precision@10) that all the tuners optimized. Also the best NDCG@10 is obtained for annealing and correlates well with precision@10. RMSE on the other hand does not correlate well and is not optimized for annealing, since finding the top k recommendations in the right order is a different task from predicting ratings (high and low) accurately. \n", + "We have also observed that the above ranking of the tuners is not consistent and may change when trying these experiments multiple times. Since some of these tuners rely heavily on randomized sampling, a larger number of trials is required to get more consistent metrics.\n", + "In addition, some of the tuning algorithms themselves come with parameters, which can affect their performance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Stop the NNI experiment \n", + "stop_nni()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tmp_dir.cleanup()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7. Concluding Remarks\n", + "\n", + "We showed how to tune **all** the hyperparameters accepted by Surprise SVD simultaneously, by utilizing the NNI toolkit. \n", + "For example, training and evaluation of a single SVD model takes about 50 seconds on the 100k MovieLens data on a Standard D2_V2 VM. Searching through 100 different combinations of hyperparameters sequentially would take about 80 minutes whereas each of the above experiments took about 10 minutes by exploiting parallelization on a single D16_v3 VM. With NNI, one can take advantage of concurrency and multiple processors on a virtual machine and can use a variety of tuning methods to navigate efficiently through a large space of hyperparameters.
\n", + "For examples of scaling larger tuning workloads on clusters of machines, see [the notebooks](./README.md) that employ the [Azure Machine Learning service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters). " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### References\n", + "\n", + "* [Matrix factorization algorithms in Surprise](https://surprise.readthedocs.io/en/stable/matrix_factorization.html) \n", + "* [Surprise SVD deep-dive notebook](../02_model/surprise_svd_deep_dive.ipynb)\n", + "* [Neural Network Intelligence toolkit](https://github.com/Microsoft/nni)" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "Python (recommenders)", + "language": "python", + "name": "recommenders" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scripts/generate_conda_file.py b/scripts/generate_conda_file.py index 7f33e97b9e..907a8be3c8 100644 --- a/scripts/generate_conda_file.py +++ b/scripts/generate_conda_file.py @@ -90,8 +90,8 @@ PIP_GPU = {"nvidia-ml-py3": "nvidia-ml-py3>=7.352.0"} PIP_PYSPARK = {"databricks-cli": "databricks-cli==0.8.6"} -PIP_DARWIN = {"nni": "nni==0.5.2.1.1"} -PIP_LINUX = {"nni": "nni==0.5.2.1.1"} +PIP_DARWIN = {"nni": "nni==1.5"} +PIP_LINUX = {"nni": "nni==1.5"} PIP_WIN32 = {} From 6e485a15dd62d6cf3d8aaadaef3cb34dc2d5917e Mon Sep 17 00:00:00 2001 From: seanytak Date: Fri, 17 Apr 2020 07:52:41 +0000 Subject: [PATCH 02/17] update nni==1.5 --- notebooks/02_model/ncf_deep_dive.ipynb | 113 ++++---- .../nni_ncf.ipynb | 4 +- .../nni_surprise_svd.ipynb | 274 +++++------------- reco_utils/tuning/nni/nni_utils.py | 8 +- 4 files changed, 134 insertions(+), 265 deletions(-) diff --git a/notebooks/02_model/ncf_deep_dive.ipynb b/notebooks/02_model/ncf_deep_dive.ipynb index 6d2a7a7a72..046c42563f 100644 --- a/notebooks/02_model/ncf_deep_dive.ipynb +++ b/notebooks/02_model/ncf_deep_dive.ipynb @@ -34,17 +34,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", - "/anaconda/envs/recommenders/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" ] }, @@ -224,7 +224,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.1kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.7kKB/s]\n" ] }, { @@ -398,7 +398,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 410.15181515900076 seconds for training.\n" + "Took 806.4961465689998 seconds for training.\n" ] } ], @@ -461,31 +461,31 @@ " 0\n", " 1.0\n", " 149.0\n", - " 0.017463\n", + " 0.047002\n", " \n", " \n", " 1\n", " 1.0\n", " 88.0\n", - " 0.407677\n", + " 0.566155\n", " \n", " \n", " 2\n", " 1.0\n", " 101.0\n", - " 0.375374\n", + " 0.226456\n", " \n", " \n", " 3\n", " 1.0\n", " 110.0\n", - " 0.026372\n", + " 0.008283\n", " \n", " \n", " 4\n", " 1.0\n", " 103.0\n", - " 0.020589\n", + " 0.005900\n", " \n", " \n", "\n", @@ -493,11 +493,11 @@ ], "text/plain": [ " userID itemID prediction\n", - "0 1.0 149.0 0.017463\n", - "1 1.0 88.0 0.407677\n", - "2 1.0 101.0 0.375374\n", - "3 1.0 110.0 0.026372\n", - "4 1.0 103.0 0.020589" + "0 1.0 149.0 0.047002\n", + "1 1.0 88.0 0.566155\n", + "2 1.0 101.0 0.226456\n", + "3 1.0 110.0 0.008283\n", + "4 1.0 103.0 0.005900" ] }, "execution_count": 8, @@ -532,7 +532,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 2.4053909909998765 seconds for prediction.\n" + "Took 2.9599234249999427 seconds for prediction.\n" ] } ], @@ -564,10 +564,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.047199\n", - "NDCG:\t0.193792\n", - "Precision@K:\t0.175504\n", - "Recall@K:\t0.097036\n" + "MAP:\t0.046569\n", + "NDCG:\t0.192938\n", + "Precision@K:\t0.173171\n", + "Recall@K:\t0.096173\n" ] } ], @@ -610,8 +610,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "HR:\t0.488684\n", - "NDCG:\t0.383331\n" + "HR:\t0.485765\n", + "NDCG:\t0.379814\n" ] } ], @@ -692,7 +692,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 332.92922498900043 seconds for training.\n" + "Took 500.5236881850001 seconds for training.\n" ] } ], @@ -735,7 +735,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 367.1780747690009 seconds for training.\n" + "Took 552.2027924220001 seconds for training.\n" ] } ], @@ -760,7 +760,16 @@ "cell_type": "code", "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Restoring parameters from .pretrain/GMF/model.ckpt\n", + "INFO:tensorflow:Restoring parameters from .pretrain/MLP/model.ckpt\n" + ] + } + ], "source": [ "model = NCF (\n", " n_users=data.n_users, \n", @@ -787,7 +796,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 414.2128918319995 seconds for training.\n" + "Took 644.399864857 seconds for training.\n" ] } ], @@ -816,7 +825,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 2.5282814109996252 seconds for prediction.\n" + "Took 3.0592451869997603 seconds for prediction.\n" ] } ], @@ -848,10 +857,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.043843\n", - "NDCG:\t0.182116\n", - "Precision@K:\t0.166702\n", - "Recall@K:\t0.093253\n" + "MAP:\t0.046286\n", + "NDCG:\t0.186660\n", + "Precision@K:\t0.169035\n", + "Recall@K:\t0.100020\n" ] } ], @@ -876,14 +885,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " \n" ] }, { "data": { "application/papermill.record+json": { - "map": 0.04719861331116455 + "map": 0.046568915592114325 } }, "metadata": {}, @@ -893,14 +902,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:3: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:3: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " This is separate from the ipykernel package so we can avoid doing imports until\n" ] }, { "data": { "application/papermill.record+json": { - "ndcg": 0.3833305241576571 + "ndcg": 0.37981413819338766 } }, "metadata": {}, @@ -910,14 +919,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " after removing the cwd from sys.path.\n" ] }, { "data": { "application/papermill.record+json": { - "precision": 0.17550371155885472 + "precision": 0.1731707317073171 } }, "metadata": {}, @@ -927,14 +936,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " \"\"\"\n" ] }, { "data": { "application/papermill.record+json": { - "recall": 0.09703642848424668 + "recall": 0.096173321399973 } }, "metadata": {}, @@ -944,14 +953,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " \n" ] }, { "data": { "application/papermill.record+json": { - "map2": 0.04384271262964714 + "map2": 0.046286433403948564 } }, "metadata": {}, @@ -961,14 +970,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " import sys\n" ] }, { "data": { "application/papermill.record+json": { - "ndcg2": 0.18211610675334622 + "ndcg2": 0.1866595787134735 } }, "metadata": {}, @@ -978,14 +987,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:8: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:8: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " \n" ] }, { "data": { "application/papermill.record+json": { - "precision2": 0.16670201484623542 + "precision2": 0.1690349946977731 } }, "metadata": {}, @@ -995,14 +1004,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/anaconda/envs/recommenders/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", " if __name__ == '__main__':\n" ] }, { "data": { "application/papermill.record+json": { - "recall2": 0.09325259530179235 + "recall2": 0.10002001904879806 } }, "metadata": {}, @@ -1065,9 +1074,9 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python (recommenders)", + "display_name": "Python (recommenders_gpu)", "language": "python", - "name": "recommenders" + "name": "recommenders_gpu" }, "language_info": { "codemirror_mode": { diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index 72afc94679..703a69a2bf 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -972,9 +972,9 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python (recommenders)", + "display_name": "Python (recommenders_gpu)", "language": "python", - "name": "recommenders" + "name": "recommenders_gpu" }, "language_info": { "codemirror_mode": { diff --git a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb index 9dfe3f54d3..1394f87386 100644 --- a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb @@ -39,10 +39,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "System version: 3.6.7 |Anaconda, Inc.| (default, Oct 23 2018, 19:16:44) \n", + "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", - "Surprise version: 1.0.6\n", - "NNI version: 0.5.2.1\n" + "Surprise version: 1.1.0\n", + "NNI version: 1.5\n" ] } ], @@ -119,7 +119,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "4.93MB [00:01, 4.03MB/s] \n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.0kKB/s]\n" ] }, { @@ -450,7 +450,29 @@ "cell_type": "code", "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 'aNZS2', 'status': 'SUCCEEDED', 'hyperParameters': ['{\"parameter_id\":0,\"parameter_source\":\"algorithm\",\"parameters\":{\"n_factors\":150,\"init_mean\":0.18393411842761564,\"init_std_dev\":0.14521525227820323,\"lr_bu\":0.09439535135482961,\"lr_bi\":0.031141612863547222,\"lr_pu\":0.01646857905973026,\"lr_qi\":0.04534025185075736,\"reg_bu\":0.6542837794618152,\"reg_bi\":0.1651950686894033,\"reg_pu\":0.150932523914943,\"reg_qi\":0.6772441234621182},\"parameter_index\":0}'], 'logPath': 'file://localhost:/tmp/tmpzp8xmmo2/experiments/dbzBCjcW/trials/aNZS2', 'startTime': 1587101415286, 'sequenceId': 0, 'endTime': 1587101416721, 'finalMetricData': [{'timestamp': 1587101416446, 'trialJobId': 'aNZS2', 'parameterId': '0', 'type': 'FINAL', 'sequence': 0, 'data': '\"{\\\\\"rmse\\\\\": 1.181392288499365, \\\\\"default\\\\\": 0.0, \\\\\"ndcg_at_k\\\\\": 0.0}\"'}]}\n", + "('{\"rmse\": 1.181392288499365, \"default\": 0.0, \"ndcg_at_k\": 0.0}', '/tmp/tmpzp8xmmo2/experiments/dbzBCjcW/trials/aNZS2')\n" + ] + }, + { + "ename": "TypeError", + "evalue": "string indices must be integers", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mcheck_metrics_written\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'maximize'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mget_trials\u001b[0;34m(optimize_mode)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m sorted_trials = sorted(\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimize_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"maximize\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m )\n\u001b[1;32m 119\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted_trials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m sorted_trials = sorted(\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimize_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"maximize\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m )\n\u001b[1;32m 119\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted_trials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: string indices must be integers" + ] + } + ], "source": [ "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", "trials, best_metrics, best_params, best_trial_path = get_trials('maximize')" @@ -458,74 +480,27 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rmse': 1.0123682114698453,\n", - " 'ndcg_at_k': 0.06785626653789574,\n", - " 'precision_at_k': 0.05867944621938233}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_metrics" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'parameter_id': 77,\n", - " 'parameter_source': 'algorithm',\n", - " 'parameters': {'n_factors': 100,\n", - " 'init_mean': 0.07836980673539604,\n", - " 'init_std_dev': 0.04625837754038721,\n", - " 'lr_bu': 0.00514802991176839,\n", - " 'lr_bi': 0.00018377384839153354,\n", - " 'lr_pu': 0.06233858917841942,\n", - " 'lr_qi': 0.08065255326258915,\n", - " 'reg_bu': 0.9535640584818021,\n", - " 'reg_bi': 0.4231926191341181,\n", - " 'reg_pu': 0.9497338396289998,\n", - " 'reg_qi': 0.7232529026643588}}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_params" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'/tmp/tmpkyzs_qxc/experiments/SXPKMSCx/trials/feUsj'" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_trial_path" ] @@ -539,7 +514,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -555,7 +530,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -573,17 +548,9 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'rmse': 1.0119432788540352, 'precision_at_k': 0.08816631130063968, 'ndcg_at_k': 0.10540817594832474}\n" - ] - } - ], + "outputs": [], "source": [ "test_results_tpe = compute_test_results(svd)\n", "print(test_results_tpe)" @@ -616,17 +583,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/tmp/tmpkyzs_qxc/config_svd.yml\n" - ] - } - ], + "outputs": [], "source": [ "# Random search\n", "config['tuner']['builtinTunerName'] = 'Random'\n", @@ -637,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -648,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -659,7 +618,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -671,7 +630,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -682,7 +641,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -693,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -705,7 +664,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -716,7 +675,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -735,7 +694,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -747,7 +706,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -763,7 +722,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -776,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -787,7 +746,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -799,7 +758,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -810,7 +769,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -828,7 +787,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -848,7 +807,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -859,7 +818,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -870,7 +829,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -885,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -909,100 +868,11 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ndcg_at_kprecision_at_krmsetime
Annealing0.1140.0971.030565.255
TPE0.1050.0881.012600.645
Hyperband0.0680.0650.958364.231
Random Search0.0510.0520.965585.224
Evolution0.0320.0331.863604.919
Metis0.0300.0300.9761506.123
\n", - "
" - ], - "text/plain": [ - " ndcg_at_k precision_at_k rmse time\n", - "Annealing 0.114 0.097 1.030 565.255\n", - "TPE 0.105 0.088 1.012 600.645\n", - "Hyperband 0.068 0.065 0.958 364.231\n", - "Random Search 0.051 0.052 0.965 585.224\n", - "Evolution 0.032 0.033 1.863 604.919\n", - "Metis 0.030 0.030 0.976 1506.123" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "res_df.sort_values(by=\"precision_at_k\", ascending=False).round(3)" ] @@ -1018,7 +888,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1028,7 +898,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1061,23 +931,11 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python (reco_bare)", + "display_name": "Python (recommenders_gpu)", "language": "python", - "name": "reco_bare" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" + "name": "recommenders_gpu" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/reco_utils/tuning/nni/nni_utils.py b/reco_utils/tuning/nni/nni_utils.py index 1eafeb3db7..d79252ff2d 100644 --- a/reco_utils/tuning/nni/nni_utils.py +++ b/reco_utils/tuning/nni/nni_utils.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import ast import json import os import requests @@ -108,7 +109,7 @@ def get_trials(optimize_mode): raise ValueError("optimize_mode should equal either minimize or maximize") all_trials = requests.get(NNI_TRIAL_JOBS_URL).json() trials = [ - (eval(trial["finalMetricData"][0]["data"]), trial["logPath"].split(":")[-1]) + (ast.literal_eval(eval(trial["finalMetricData"][0]["data"])), trial["logPath"].split(":")[-1]) for trial in all_trials ] sorted_trials = sorted( @@ -141,8 +142,9 @@ def start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES): max_retries (int): max number of retries """ nni_env = os.environ.copy() - nni_env["PATH"] = sys.prefix + "/bin:" + nni_env["PATH"] - proc = subprocess.run(["nnictl", "create", "--config", config_path], env=nni_env) + nni_env['PATH'] = sys.prefix + '/bin:' + nni_env['PATH'] + proc = subprocess.run([sys.prefix + '/bin/nnictl', 'create', '--config', config_path], env=nni_env) + # proc = subprocess.run(["nnictl", "create", "--config", config_path], env=nni_env) if proc.returncode != 0: raise RuntimeError("'nnictl create' failed with code %d" % proc.returncode) check_experiment_status(wait=wait, max_retries=max_retries) From 67e54deb53f823891e5b938f3785d6a82ea68771 Mon Sep 17 00:00:00 2001 From: seanytak Date: Fri, 17 Apr 2020 07:55:36 +0000 Subject: [PATCH 03/17] update nni surprise svd for nni 1.5 --- .../nni_surprise_svd.ipynb | 93 ++++++++++++------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb index 1394f87386..6b819468a7 100644 --- a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 59, "metadata": {}, "outputs": [ { @@ -42,7 +42,9 @@ "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", "Surprise version: 1.1.0\n", - "NNI version: 1.5\n" + "NNI version: 1.5\n", + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" ] } ], @@ -90,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 60, "metadata": { "tags": [ "parameters" @@ -112,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -192,7 +194,7 @@ "4 166 346 1.0" ] }, - "execution_count": 3, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -208,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 62, "metadata": {}, "outputs": [], "source": [ @@ -217,7 +219,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 63, "metadata": { "scrolled": true }, @@ -264,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 64, "metadata": {}, "outputs": [], "source": [ @@ -305,7 +307,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -329,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -346,7 +348,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -401,7 +403,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 68, "metadata": {}, "outputs": [], "source": [ @@ -411,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -425,7 +427,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -448,28 +450,22 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 71, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': 'aNZS2', 'status': 'SUCCEEDED', 'hyperParameters': ['{\"parameter_id\":0,\"parameter_source\":\"algorithm\",\"parameters\":{\"n_factors\":150,\"init_mean\":0.18393411842761564,\"init_std_dev\":0.14521525227820323,\"lr_bu\":0.09439535135482961,\"lr_bi\":0.031141612863547222,\"lr_pu\":0.01646857905973026,\"lr_qi\":0.04534025185075736,\"reg_bu\":0.6542837794618152,\"reg_bi\":0.1651950686894033,\"reg_pu\":0.150932523914943,\"reg_qi\":0.6772441234621182},\"parameter_index\":0}'], 'logPath': 'file://localhost:/tmp/tmpzp8xmmo2/experiments/dbzBCjcW/trials/aNZS2', 'startTime': 1587101415286, 'sequenceId': 0, 'endTime': 1587101416721, 'finalMetricData': [{'timestamp': 1587101416446, 'trialJobId': 'aNZS2', 'parameterId': '0', 'type': 'FINAL', 'sequence': 0, 'data': '\"{\\\\\"rmse\\\\\": 1.181392288499365, \\\\\"default\\\\\": 0.0, \\\\\"ndcg_at_k\\\\\": 0.0}\"'}]}\n", - "('{\"rmse\": 1.181392288499365, \"default\": 0.0, \"ndcg_at_k\": 0.0}', '/tmp/tmpzp8xmmo2/experiments/dbzBCjcW/trials/aNZS2')\n" - ] - }, - { - "ename": "TypeError", - "evalue": "string indices must be integers", + "ename": "ValueError", + "evalue": "malformed node or string: {'rmse': 1.160248985195997, 'default': 0, 'ndcg_at_k': 0}", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mcheck_metrics_written\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'maximize'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mget_trials\u001b[0;34m(optimize_mode)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m sorted_trials = sorted(\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimize_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"maximize\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m )\n\u001b[1;32m 119\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted_trials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m sorted_trials = sorted(\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimize_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"maximize\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m )\n\u001b[1;32m 119\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted_trials\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: string indices must be integers" + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mcheck_metrics_written\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'maximize'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mget_trials\u001b[0;34m(optimize_mode)\u001b[0m\n\u001b[1;32m 111\u001b[0m trials = [\n\u001b[1;32m 112\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"finalMetricData\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"logPath\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_trials\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 114\u001b[0m ]\n\u001b[1;32m 115\u001b[0m sorted_trials = sorted(\n", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 111\u001b[0m trials = [\n\u001b[1;32m 112\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"finalMetricData\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"logPath\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_trials\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 114\u001b[0m ]\n\u001b[1;32m 115\u001b[0m sorted_trials = sorted(\n", + "\u001b[0;32m/anaconda/envs/recommenders_gpu/lib/python3.6/ast.py\u001b[0m in \u001b[0;36mliteral_eval\u001b[0;34m(node_or_string)\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'malformed node or string: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/anaconda/envs/recommenders_gpu/lib/python3.6/ast.py\u001b[0m in \u001b[0;36m_convert\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'malformed node or string: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 85\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: malformed node or string: {'rmse': 1.160248985195997, 'default': 0, 'ndcg_at_k': 0}" ] } ], @@ -583,22 +579,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ "# Random search\n", "config['tuner']['builtinTunerName'] = 'Random'\n", + "if 'classArgs' in config['tuner']:\n", + " config['tuner'].pop('classArgs')\n", + " \n", "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))\n", - " print(config_path)" + " fp.write(yaml.dump(config, default_flow_style=False))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 87, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CompletedProcess(args=['/anaconda/envs/recommenders_gpu/bin/nnictl', 'create', '--config', '/tmp/tmpxs86uz3d/config_svd.yml'], returncode=0)\n" + ] + } + ], "source": [ "stop_nni()\n", "with Timer() as time_random:\n", @@ -624,6 +630,11 @@ "source": [ "# Annealing\n", "config['tuner']['builtinTunerName'] = 'Anneal'\n", + "if 'classArgs' not in config['tuner']:\n", + " config['tuner']['classArgs'] = {'optimize_mode': 'maximum'}\n", + "else:\n", + " config['tuner']['classArgs']['optimize_mode'] = 'maximum'\n", + " \n", "with open(config_path, 'w') as fp:\n", " fp.write(yaml.dump(config, default_flow_style=False))" ] @@ -934,6 +945,18 @@ "display_name": "Python (recommenders_gpu)", "language": "python", "name": "recommenders_gpu" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" } }, "nbformat": 4, From f20cf2e7091b85650ef750743fd313ce2bc1edd9 Mon Sep 17 00:00:00 2001 From: seanytak Date: Tue, 21 Apr 2020 23:27:15 +0000 Subject: [PATCH 04/17] add ncf training using nni --- reco_utils/tuning/nni/ncf_training.py | 188 ++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 reco_utils/tuning/nni/ncf_training.py diff --git a/reco_utils/tuning/nni/ncf_training.py b/reco_utils/tuning/nni/ncf_training.py new file mode 100644 index 0000000000..e00d87967a --- /dev/null +++ b/reco_utils/tuning/nni/ncf_training.py @@ -0,0 +1,188 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import json +import logging +import numpy as np +import os +import pandas as pd +import nni +import surprise +import sys + +sys.path.append("../../../") + +import reco_utils.evaluation.python_evaluation as evaluation +from reco_utils.recommender.surprise.surprise_utils import ( + predict, + compute_ranking_predictions, +) + +from reco_utils.recommender.ncf.ncf_singlenode import NCF +from reco_utils.recommender.ncf.dataset import Dataset as NCFDataset +from reco_utils.dataset import movielens +from reco_utils.dataset.python_splitters import python_chrono_split +from reco_utils.evaluation.python_evaluation import (rmse, mae, rsquared, exp_var, map_at_k, ndcg_at_k, precision_at_k, + recall_at_k, get_top_k_items) +from reco_utils.common.constants import SEED as DEFAULT_SEED + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("ncf") + + +def ncf_training(params): + """ + Train NCF using the given hyper-parameters + """ + logger.debug("Start training...") + train_data = pd.read_pickle( + path=os.path.join(params["datastore"], params["train_datapath"]) + ) + validation_data = pd.read_pickle( + path=os.path.join(params["datastore"], params["validation_datapath"]) + ) + + data = NCFDataset(train=train_data, test=validation_data, seed=DEFAULT_SEED) + + ncf_params = { + p: params[p] + for p in [ + "random_state", + "n_epochs", + "verbose", + "n_factors", + "learning_rate", + ] + } + + model = NCF ( + n_users=data.n_users, + n_items=data.n_items, + model_type="NeuMF", + n_factors=params["n_factors"], + layer_sizes=[16,8,4], + n_epochs=params["n_epochs"], + learning_rate=params["learning_rate"], + verbose=params["verbose"], + seed=DEFAULT_SEED + ) + + model.fit(data) + + logger.debug("Evaluating...") + + metrics_dict = {} + rating_metrics = params["rating_metrics"] + if len(rating_metrics) > 0: + predictions = [[row.userID, row.itemID, model.predict(row.userID, row.itemID)] + for (_, row) in validation_data.iterrows()] + + predictions = pd.DataFrame(predictions, columns=['userID', 'itemID', 'prediction']) + predictions = predictions.astype({'userID': 'int64', 'itemID': 'int64', 'prediction': 'float64'}) + + for metric in rating_metrics: + result = getattr(evaluation, metric)(validation_data, predictions) + logger.debug("%s = %g", metric, result) + if metric == params["primary_metric"]: + metrics_dict["default"] = result + else: + metrics_dict[metric] = result + + ranking_metrics = params["ranking_metrics"] + if len(ranking_metrics) > 0: + users, items, preds = [], [], [] + item = list(train_data.itemID.unique()) + for user in train_data.userID.unique(): + user = [user] * len(item) + users.extend(user) + items.extend(item) + preds.extend(list(model.predict(user, item, is_list=True))) + + all_predictions = pd.DataFrame(data={"userID": users, "itemID": items, "prediction": preds}) + + merged = pd.merge(train_data, all_predictions, on=["userID", "itemID"], how="outer") + all_predictions = merged[merged.rating.isnull()].drop('rating', axis=1) + for metric in ranking_metrics: + result = getattr(evaluation, metric)( + validation_data, all_predictions, col_prediction="prediction", k=params["k"] + ) + logger.debug("%s@%d = %g", metric, params["k"], result) + if metric == params["primary_metric"]: + metrics_dict["default"] = result + else: + metrics_dict[metric] = result + + if len(ranking_metrics) == 0 and len(rating_metrics) == 0: + raise ValueError("No metrics were specified.") + + # Report the metrics + nni.report_final_result(metrics_dict) + + # Save the metrics in a JSON file + output_dir = os.environ.get("NNI_OUTPUT_DIR") + with open(os.path.join(output_dir, "metrics.json"), "w") as fp: + temp_dict = metrics_dict.copy() + temp_dict[params["primary_metric"]] = temp_dict.pop("default") + json.dump(temp_dict, fp) + + return model + + +def get_params(): + parser = argparse.ArgumentParser() + # Data path + parser.add_argument( + "--datastore", type=str, dest="datastore", help="Datastore path" + ) + parser.add_argument("--train-datapath", type=str, dest="train_datapath") + parser.add_argument("--validation-datapath", type=str, dest="validation_datapath") + parser.add_argument("--surprise-reader", type=str, dest="surprise_reader") + parser.add_argument("--usercol", type=str, dest="usercol", default="userID") + parser.add_argument("--itemcol", type=str, dest="itemcol", default="itemID") + # Metrics + parser.add_argument( + "--rating-metrics", type=str, nargs="*", dest="rating_metrics", default=[] + ) + parser.add_argument( + "--ranking-metrics", type=str, nargs="*", dest="ranking_metrics", default=[] + ) + parser.add_argument("--k", type=int, dest="k", default=None) + parser.add_argument("--remove-seen", dest="remove_seen", action="store_false") + # Training parameters + parser.add_argument("--random-state", type=int, dest="random_state", default=0) + parser.add_argument("--verbose", dest="verbose", action="store_true") + parser.add_argument("--epochs", type=int, dest="n_epochs", default=30) + parser.add_argument("--biased", dest="biased", action="store_true") + parser.add_argument("--primary-metric", dest="primary_metric", default="rmse") + # Hyperparameters to be tuned + parser.add_argument("--n_factors", type=int, dest="n_factors", default=100) + + args = parser.parse_args() + return args + + +def main(params): + logger.debug("Args: %s", str(params)) + logger.debug("Number of epochs %d", params["n_epochs"]) + + model = ncf_training(params) + # Save NCF model to the output directory for later use + output_dir = os.environ.get("NNI_OUTPUT_DIR") + # surprise.dump.dump(os.path.join(output_dir, "model.dump"), algo=svd) + + +if __name__ == "__main__": + try: + tuner_params = nni.get_next_parameter() + logger.debug("Hyperparameters: %s", tuner_params) + params = vars(get_params()) + # in the case of Hyperband, use STEPS to allocate the number of epochs SVD will run for + if "STEPS" in tuner_params: + steps_param = tuner_params["STEPS"] + params["n_epochs"] = int(np.rint(steps_param)) + params.update(tuner_params) + main(params) + except Exception as exception: + logger.exception(exception) + raise From 457d363bd7bafb869aed99a576c2eb629b958ca0 Mon Sep 17 00:00:00 2001 From: seanytak Date: Tue, 21 Apr 2020 23:27:40 +0000 Subject: [PATCH 05/17] add ncf tuning harness notebook --- .../nni_ncf.ipynb | 306 +++++++++++------- 1 file changed, 193 insertions(+), 113 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index 703a69a2bf..109cf8b719 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -7,14 +7,14 @@ "Copyright (c) Microsoft Corporation. All rights reserved.
\n", "Licensed under the MIT License.\n", "
\n", - "# Hyperparameter Tuning for Matrix Factorization Using the Neural Network Intelligence Toolkit" + "# Model Comparison between SVD and NCF Using the Neural Network Intelligence Toolkit" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters of a matrix factorization model. In particular, we optimize the hyperparameters of [Surprise SVD](https://surprise.readthedocs.io/en/stable/matrix_factorization.html).\n", + "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters for the Neural Collaborative Filtering Model. In particular, we optimize the hyperparameters of [Surprise SVD](https://surprise.readthedocs.io/en/stable/matrix_factorization.html).\n", "\n", "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. NNI comes with [several tuning algorithms](https://nni.readthedocs.io/en/latest/Builtin_Tuner.html) built in. It also allows users to [define their own general purpose tuners](https://nni.readthedocs.io/en/latest/Customize_Tuner.html). NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", "\n", @@ -35,6 +35,24 @@ "execution_count": 1, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", + "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -42,7 +60,7 @@ "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", "Surprise version: 1.1.0\n", - "NNI version: 0.5.2.1.1\n" + "NNI version: 1.5\n" ] } ], @@ -56,6 +74,7 @@ "import pandas as pd\n", "import shutil\n", "import subprocess\n", + "import tensorflow as tf\n", "import yaml\n", "import pkg_resources\n", "from tempfile import TemporaryDirectory\n", @@ -69,6 +88,14 @@ " stop_nni, start_nni)\n", "from reco_utils.recommender.surprise.surprise_utils import predict, compute_ranking_predictions\n", "\n", + "from reco_utils.recommender.ncf.ncf_singlenode import NCF\n", + "from reco_utils.recommender.ncf.dataset import Dataset as NCFDataset\n", + "from reco_utils.dataset import movielens\n", + "from reco_utils.dataset.python_splitters import python_chrono_split\n", + "from reco_utils.evaluation.python_evaluation import (rmse, mae, rsquared, exp_var, map_at_k, ndcg_at_k, precision_at_k, \n", + " recall_at_k, get_top_k_items)\n", + "from reco_utils.common.constants import SEED as DEFAULT_SEED\n", + "\n", "print(\"System version: {}\".format(sys.version))\n", "print(\"Surprise version: {}\".format(surprise.__version__))\n", "print(\"NNI version: {}\".format(pkg_resources.get_distribution(\"nni\").version))\n", @@ -104,10 +131,10 @@ "SURPRISE_READER = 'ml-100k'\n", "TMP_DIR = tmp_dir.name\n", "NUM_EPOCHS = 30\n", - "MAX_TRIAL_NUM = 100\n", + "MAX_TRIAL_NUM = 10\n", "# time (in seconds) to wait for each tuning experiment to complete\n", "WAITING_TIME = 20\n", - "MAX_RETRIES = 400 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" + "MAX_RETRIES = MAX_TRIAL_NUM*4 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" ] }, { @@ -119,7 +146,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.5kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 7.55kKB/s]\n" ] }, { @@ -146,6 +173,7 @@ " userID\n", " itemID\n", " rating\n", + " timestamp\n", " \n", " \n", " \n", @@ -154,42 +182,47 @@ " 196\n", " 242\n", " 3.0\n", + " 881250949\n", " \n", " \n", " 1\n", " 186\n", " 302\n", " 3.0\n", + " 891717742\n", " \n", " \n", " 2\n", " 22\n", " 377\n", " 1.0\n", + " 878887116\n", " \n", " \n", " 3\n", " 244\n", " 51\n", " 2.0\n", + " 880606923\n", " \n", " \n", " 4\n", " 166\n", " 346\n", " 1.0\n", + " 886397596\n", " \n", " \n", "\n", "" ], "text/plain": [ - " userID itemID rating\n", - "0 196 242 3.0\n", - "1 186 302 3.0\n", - "2 22 377 1.0\n", - "3 244 51 2.0\n", - "4 166 346 1.0" + " userID itemID rating timestamp\n", + "0 196 242 3.0 881250949\n", + "1 186 302 3.0 891717742\n", + "2 22 377 1.0 878887116\n", + "3 244 51 2.0 880606923\n", + "4 166 346 1.0 886397596" ] }, "execution_count": 3, @@ -198,12 +231,13 @@ } ], "source": [ - "data = movielens.load_pandas_df(\n", + "# Note: The NCF model can incorporate\n", + "df = movielens.load_pandas_df(\n", " size=MOVIELENS_DATA_SIZE,\n", - " header=[\"userID\", \"itemID\", \"rating\"]\n", - ")\n", + " header=[\"userID\", \"itemID\", \"rating\", \"timestamp\"]\n", + ")[:100]\n", "\n", - "data.head()" + "df.head()" ] }, { @@ -212,7 +246,29 @@ "metadata": {}, "outputs": [], "source": [ - "train, validation, test = python_random_split(data, [0.7, 0.15, 0.15])" + "train, test = python_chrono_split(df, 0.75)\n", + "train = train.drop(['timestamp'], axis=1)\n", + "test = test.drop(['timestamp'], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "userID int64\n", + "itemID int64\n", + "rating float64\n", + "dtype: object\n" + ] + } + ], + "source": [ + "print(test.dtypes)" ] }, { @@ -232,9 +288,6 @@ "TRAIN_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_train.pkl\"\n", "train.to_pickle(os.path.join(DATA_DIR, TRAIN_FILE_NAME))\n", "\n", - "VAL_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_val.pkl\"\n", - "validation.to_pickle(os.path.join(DATA_DIR, VAL_FILE_NAME))\n", - "\n", "TEST_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_test.pkl\"\n", "test.to_pickle(os.path.join(DATA_DIR, TEST_FILE_NAME))" ] @@ -268,7 +321,6 @@ "metadata": {}, "outputs": [], "source": [ - "EXP_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_svd_model\"\n", "PRIMARY_METRIC = \"precision_at_k\"\n", "RATING_METRICS = [\"rmse\"]\n", "RANKING_METRICS = [\"precision_at_k\", \"ndcg_at_k\"] \n", @@ -283,7 +335,7 @@ "script_params = \" \".join([\n", " \"--datastore\", DATA_DIR,\n", " \"--train-datapath\", TRAIN_FILE_NAME,\n", - " \"--validation-datapath\", VAL_FILE_NAME,\n", + " \"--validation-datapath\", TEST_FILE_NAME,\n", " \"--surprise-reader\", SURPRISE_READER,\n", " \"--rating-metrics\", \" \".join(RATING_METRICS),\n", " \"--ranking-metrics\", \" \".join(RANKING_METRICS),\n", @@ -312,7 +364,7 @@ "# hyperparameters search space\n", "# We do not set 'lr_all' and 'reg_all' because they will be overriden by the other lr_ and reg_ parameters\n", "\n", - "hyper_params = {\n", + "svd_hyper_params = {\n", " 'n_factors': {\"_type\": \"choice\", \"_value\": [10, 50, 100, 150, 200]},\n", " 'init_mean': {\"_type\": \"uniform\", \"_value\": [-0.5, 0.5]},\n", " 'init_std_dev': {\"_type\": \"uniform\", \"_value\": [0.01, 0.2]},\n", @@ -334,7 +386,7 @@ "outputs": [], "source": [ "with open(os.path.join(TMP_DIR, 'search_space_svd.json'), 'w') as fp:\n", - " json.dump(hyper_params, fp)" + " json.dump(svd_hyper_params, fp)" ] }, { @@ -380,6 +432,64 @@ " fp.write(yaml.dump(config, default_flow_style=False))" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "ncf_hyper_params = {\n", + " 'n_factors': {\"_type\": \"choice\", \"_value\": [8, 12, 16, 24, 40]},\n", + " 'learning_rate': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]},\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "with open(os.path.join(TMP_DIR, 'search_space_ncf.json'), 'w') as fp:\n", + " json.dump(ncf_hyper_params, fp)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "config = {\n", + " \"authorName\": \"default\",\n", + " \"experimentName\": \"tensorflow_ncf\",\n", + " \"trialConcurrency\": 8,\n", + " \"maxExecDuration\": \"1h\",\n", + " \"maxTrialNum\": MAX_TRIAL_NUM,\n", + " \"trainingServicePlatform\": \"local\",\n", + " # The path to Search Space\n", + " \"searchSpacePath\": \"search_space_ncf.json\",\n", + " \"useAnnotation\": False,\n", + " \"logDir\": LOG_DIR,\n", + " \"tuner\": {\n", + " \"builtinTunerName\": \"TPE\",\n", + " \"classArgs\": {\n", + " #choice: maximize, minimize\n", + " \"optimize_mode\": \"maximize\"\n", + " }\n", + " },\n", + " # The path and the running command of trial\n", + " \"trial\": {\n", + " \"command\": sys.prefix + \"/bin/python ncf_training.py\" + \" \" + script_params,\n", + " \"codeDir\": os.path.join(os.path.split(os.path.abspath(reco_utils.__file__))[0], \"tuning\", \"nni\"),\n", + " \"gpuNum\": 0\n", + " }\n", + "}\n", + " \n", + "with open(os.path.join(TMP_DIR, \"config_ncf.yml\"), \"w\") as fp:\n", + " fp.write(yaml.dump(config, default_flow_style=False))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -401,36 +511,69 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop_nni\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mconfig_path_svd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTMP_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'config_svd.yml'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mTimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtime_svd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mstart_nni\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig_path_svd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mstop_nni\u001b[0;34m()\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'nnictl stop' failed with code %d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m \u001b[0mcheck_stopped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mcheck_stopped\u001b[0;34m(wait, max_retries)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 73\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "stop_nni()\n", + "config_path_svd = os.path.join(TMP_DIR, 'config_svd.yml')\n", + "with Timer() as time_svd:\n", + " start_nni(config_path_svd, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# Make sure that there is no experiment running\n", - "stop_nni()" + "print(TMP_DIR)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "config_path = os.path.join(TMP_DIR, 'config_svd.yml')\n", - "nni_env = os.environ.copy()\n", - "nni_env['PATH'] = sys.prefix + '/bin:' + nni_env['PATH']\n", - "proc = subprocess.run([sys.prefix + '/bin/nnictl', 'create', '--config', config_path], env=nni_env)\n", - "if proc.returncode != 0:\n", - " raise RuntimeError(\"'nnictl create' failed with code %d\" % proc.returncode)" + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "trials_svd, best_metrics_svd, best_params_svd, best_trial_path_svd = get_trials('maximize')" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stop_nni()\n", + "config_path_ncf = os.path.join(TMP_DIR, 'config_ncf.yml')\n", + "with Timer() as time_ncf:\n", + " start_nni(config_path_ncf, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "with Timer() as time_tpe:\n", - " check_experiment_status(wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "trials_ncf, best_metrics_ncf, best_params_ncf, best_trial_path_ncf = get_trials('maximize')" ] }, { @@ -448,7 +591,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -458,74 +601,27 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rmse': 1.0148825026253758,\n", - " 'ndcg_at_k': 0.07073059758480135,\n", - " 'precision_at_k': 0.05867944621938233}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_metrics" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'parameter_id': 20,\n", - " 'parameter_source': 'algorithm',\n", - " 'parameters': {'n_factors': 10,\n", - " 'init_mean': -0.20640543895286745,\n", - " 'init_std_dev': 0.041877648647603426,\n", - " 'lr_bu': 0.010258651938050185,\n", - " 'lr_bi': 8.286686919368394e-05,\n", - " 'lr_pu': 0.09981032112256295,\n", - " 'lr_qi': 0.06900431839934824,\n", - " 'reg_bu': 0.012030372162218295,\n", - " 'reg_bi': 0.10285996563173667,\n", - " 'reg_pu': 0.21894910939861875,\n", - " 'reg_qi': 0.8864729891838474}}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_params" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'/tmp/tmp1542zrho/experiments/oV4pQDZF/trials/uxVdI'" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_trial_path" ] @@ -539,7 +635,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -555,7 +651,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -573,17 +669,9 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'rmse': 1.0119318175407621, 'precision_at_k': 0.09872068230277187, 'ndcg_at_k': 0.11604975035480133}\n" - ] - } - ], + "outputs": [], "source": [ "test_results_tpe = compute_test_results(svd)\n", "print(test_results_tpe)" @@ -616,17 +704,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/tmp/tmp1542zrho/config_svd.yml\n" - ] - } - ], + "outputs": [], "source": [ "# Random search\n", "config['tuner']['builtinTunerName'] = 'Random'\n", From 7a4f24e2df9945b1ba82758757917d95befd4d8f Mon Sep 17 00:00:00 2001 From: seanytak Date: Wed, 22 Apr 2020 21:48:54 +0000 Subject: [PATCH 06/17] add model comparison for svd and ncf --- .../nni_ncf.ipynb | 592 ++++++------------ 1 file changed, 178 insertions(+), 414 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index 109cf8b719..a8a9fc6936 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -14,11 +14,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters for the Neural Collaborative Filtering Model. In particular, we optimize the hyperparameters of [Surprise SVD](https://surprise.readthedocs.io/en/stable/matrix_factorization.html).\n", + "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters for the Neural Collaborative Filtering Model and Surprise SVD model.\n", "\n", - "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. NNI comes with [several tuning algorithms](https://nni.readthedocs.io/en/latest/Builtin_Tuner.html) built in. It also allows users to [define their own general purpose tuners](https://nni.readthedocs.io/en/latest/Customize_Tuner.html). NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", + "To learn about each tuner NNI offers you can read about it [here](https://nni.readthedocs.io/en/latest/Tuner/BuiltinTuner.html). To see how each tuner performs on the Surprise SVD model, visit [this notebook instead](./nni_surprise_svd.ipynb). \n", "\n", - "In this notebook we execute several NNI _experiments_ on the same data sets obtained from Movielens with a training-validation-test split. Each experiment corresponds to one of the built-in tuning algorithms. It consists of many parallel _trials_, each of which corresponds to a choice of hyperparameters sampled by the tuning algorithm. All the experiments require a call to the same [python script](../../reco_utils/nni/svd_training.py) for training the SVD model and evaluating rating and ranking metrics on the test data. This script has been adapted from the [Surprise SVD notebook](../02_model/surprise_svd_deep_dive.ipynb) with only a few changes. In all experiments, we maximize precision@10. \n", + "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. . NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", + "\n", + "In this notebook, we can see how NNI works with two different model types and the differences between their hyperparameter search spaces, yaml config file, and training scripts.\n", + "\n", + "- [Surprise SVD Training Script](../../reco_utils/nni/svd_training.py)\n", + "- [NCF Training Script](../../reco_utils/nni/ncf_training.py)\n", + "\n", + "In all experiments, we maximize precision@10. \n", "\n", "For this notebook we use a _local machine_ as the training platform (this can be any machine running the `reco_base` conda environment). In this case, NNI uses the available processors of the machine to parallelize the trials, subject to the value of `trialConcurrency` we specify in the configuration. Our runs and the results we report were obtained on a [Standard_D16_v3 virtual machine](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/sizes-general#dv3-series-1) with 16 vcpus and 64 GB memory." ] @@ -82,18 +89,15 @@ "import reco_utils\n", "from reco_utils.common.timer import Timer\n", "from reco_utils.dataset import movielens\n", - "from reco_utils.dataset.python_splitters import python_random_split\n", + "from reco_utils.dataset.python_splitters import python_chrono_split\n", "from reco_utils.evaluation.python_evaluation import rmse, precision_at_k, ndcg_at_k\n", "from reco_utils.tuning.nni.nni_utils import (check_experiment_status, check_stopped, check_metrics_written, get_trials,\n", " stop_nni, start_nni)\n", - "from reco_utils.recommender.surprise.surprise_utils import predict, compute_ranking_predictions\n", - "\n", - "from reco_utils.recommender.ncf.ncf_singlenode import NCF\n", "from reco_utils.recommender.ncf.dataset import Dataset as NCFDataset\n", - "from reco_utils.dataset import movielens\n", - "from reco_utils.dataset.python_splitters import python_chrono_split\n", - "from reco_utils.evaluation.python_evaluation import (rmse, mae, rsquared, exp_var, map_at_k, ndcg_at_k, precision_at_k, \n", - " recall_at_k, get_top_k_items)\n", + "from reco_utils.recommender.ncf.ncf_singlenode import NCF\n", + "from reco_utils.recommender.surprise.surprise_utils import predict, compute_ranking_predictions\n", + "# from reco_utils.evaluation.python_evaluation import (rmse, mae, rsquared, exp_var, map_at_k, ndcg_at_k, precision_at_k, \n", + "# recall_at_k, get_top_k_items)\n", "from reco_utils.common.constants import SEED as DEFAULT_SEED\n", "\n", "print(\"System version: {}\".format(sys.version))\n", @@ -131,7 +135,7 @@ "SURPRISE_READER = 'ml-100k'\n", "TMP_DIR = tmp_dir.name\n", "NUM_EPOCHS = 30\n", - "MAX_TRIAL_NUM = 10\n", + "MAX_TRIAL_NUM = 30\n", "# time (in seconds) to wait for each tuning experiment to complete\n", "WAITING_TIME = 20\n", "MAX_RETRIES = MAX_TRIAL_NUM*4 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" @@ -146,7 +150,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 7.55kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.0kKB/s]\n" ] }, { @@ -235,7 +239,7 @@ "df = movielens.load_pandas_df(\n", " size=MOVIELENS_DATA_SIZE,\n", " header=[\"userID\", \"itemID\", \"rating\", \"timestamp\"]\n", - ")[:100]\n", + ")\n", "\n", "df.head()" ] @@ -246,31 +250,12 @@ "metadata": {}, "outputs": [], "source": [ - "train, test = python_chrono_split(df, 0.75)\n", + "train, validation, test = python_chrono_split(df, [0.7, 0.15, 0.15])\n", "train = train.drop(['timestamp'], axis=1)\n", + "validation = validation.drop(['timestamp'], axis=1)\n", "test = test.drop(['timestamp'], axis=1)" ] }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "userID int64\n", - "itemID int64\n", - "rating float64\n", - "dtype: object\n" - ] - } - ], - "source": [ - "print(test.dtypes)" - ] - }, { "cell_type": "code", "execution_count": 5, @@ -288,6 +273,9 @@ "TRAIN_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_train.pkl\"\n", "train.to_pickle(os.path.join(DATA_DIR, TRAIN_FILE_NAME))\n", "\n", + "VAL_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_val.pkl\"\n", + "validation.to_pickle(os.path.join(DATA_DIR, VAL_FILE_NAME))\n", + "\n", "TEST_FILE_NAME = \"movielens_\" + MOVIELENS_DATA_SIZE + \"_test.pkl\"\n", "test.to_pickle(os.path.join(DATA_DIR, TEST_FILE_NAME))" ] @@ -303,9 +291,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We now prepare a training script [svd_training_nni.py](../../reco_utils/nni/svd_training.py) for the hyperparameter tuning, which will log our target metrics such as precision, NDCG, RMSE.\n", - "We define the arguments of the script and the search space for the hyperparameters. All the parameter values will be passed to our training script.
\n", - "Note that we specify _precision@10_ as the primary metric. We will also instruct NNI (in the configuration file) to _maximize_ the primary metric. This is passed as an argument in the training script and the evaluated metric is returned through the NNI python library. In addition, we also evaluate RMSE and NDCG@10. " + "To run an experiment on NNI we require a general training script for our model of choice.\n", + "A general framework for a training script utilizes the following components\n", + "1. Argument Parse for the fixed parameters (dataset location, metrics to use)\n", + "2. Data preprocessing steps specific to the model\n", + "3. Fitting the model on the train set\n", + "4. Evaluating the model on the validation set on each metric (ranking and rating)\n", + "5. Save metrics and model\n", + "\n", + "To utilize NNI we also require a hypeyparameter search space. Only the hyperparameters we want to tune are required in the dictionary. NNI supports different methods of [hyperparameter sampling](https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html)." ] }, { @@ -335,7 +329,7 @@ "script_params = \" \".join([\n", " \"--datastore\", DATA_DIR,\n", " \"--train-datapath\", TRAIN_FILE_NAME,\n", - " \"--validation-datapath\", TEST_FILE_NAME,\n", + " \"--validation-datapath\", VAL_FILE_NAME,\n", " \"--surprise-reader\", SURPRISE_READER,\n", " \"--rating-metrics\", \" \".join(RATING_METRICS),\n", " \"--ranking-metrics\", \" \".join(RANKING_METRICS),\n", @@ -432,6 +426,13 @@ " fp.write(yaml.dump(config, default_flow_style=False))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We specify the search space for the NCF hyperparameters" + ] + }, { "cell_type": "code", "execution_count": 10, @@ -454,6 +455,16 @@ " json.dump(ncf_hyper_params, fp)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our NCF config.yml file follows the same structure as the SVD config.yml. The only differences are the\n", + "- Experiment name\n", + "- Hyperparameter Search Space\n", + "- The executed command for the trial (Note: The script parameters have been configured to work for both training scripts)" + ] + }, { "cell_type": "code", "execution_count": 12, @@ -498,36 +509,19 @@ "\n", "The conda environment comes with NNI installed, which includes the command line tool `nnictl` for controlling and getting information about NNI experiments.
\n", "To start the NNI tuning trials from the command line, execute the following command:
\n", - "`nnictl create --config `
\n", - "In the cell below, we call this command programmatically.
\n", - "You can see the progress of the experiment by using the URL links output by the above command.\n", + "`nnictl create --config `
\n", "\n", - "![](https://recodatasets.blob.core.windows.net/images/nn1.png)\n", "\n", - "![](https://recodatasets.blob.core.windows.net/images/nn2.png)\n", + "The `start_nni` function will run the `nnictl create` command. To find the URL for an active experiment you can run `nnictl webui url` on your terminal.\n", "\n", - "![](https://recodatasets.blob.core.windows.net/images/nn3.png)" + "In this notebook the SVD and NCF models are trained sequentially on different NNI experiments. While NNI can run two separate experiments simultaneously by adding the `--port ` flag to `nnictl create`, the total training time will probably be the same as running the experiments sequentially since these are CPU bound processes." ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, - "outputs": [ - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstop_nni\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mconfig_path_svd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTMP_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'config_svd.yml'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mTimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtime_svd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mstart_nni\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig_path_svd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mstop_nni\u001b[0;34m()\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'nnictl stop' failed with code %d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m \u001b[0mcheck_stopped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mcheck_stopped\u001b[0;34m(wait, max_retries)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 73\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "stop_nni()\n", "config_path_svd = os.path.join(TMP_DIR, 'config_svd.yml')\n", @@ -537,16 +531,7 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(TMP_DIR)" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -556,197 +541,57 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stop_nni()\n", - "config_path_ncf = os.path.join(TMP_DIR, 'config_ncf.yml')\n", - "with Timer() as time_ncf:\n", - " start_nni(config_path_ncf, wait=WAITING_TIME, max_retries=MAX_RETRIES)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "trials_ncf, best_metrics_ncf, best_params_ncf, best_trial_path_ncf = get_trials('maximize')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5. Show Results\n", - "\n", - "The trial with the best metric and the corresponding metrics and hyperparameters can also be read from the Web UI\n", - "\n", - "![](https://recodatasets.blob.core.windows.net/images/nni4.png)\n", - "\n", - "or from the JSON file created by the training script. Below, we do this programmatically using [nni_utils.py](../../reco_utils/nni/nni_utils.py) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "trials, best_metrics, best_params, best_trial_path = get_trials('maximize')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_metrics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_params" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_trial_path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This directory path is where info about the trial can be found, including logs, parameters and the model that was learned. To evaluate the metrics on the test data, we get the SVD model that was saved as `model.dump` in the training script." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "svd = surprise.dump.load(os.path.join(best_trial_path, \"model.dump\"))[1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following function computes all the metrics given an SVD model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def compute_test_results(svd):\n", - " test_results = {}\n", - " predictions = predict(svd, test, usercol=\"userID\", itemcol=\"itemID\")\n", - " for metric in RATING_METRICS:\n", - " test_results[metric] = eval(metric)(test, predictions)\n", - "\n", - " all_predictions = compute_ranking_predictions(svd, train, usercol=\"userID\", itemcol=\"itemID\", remove_seen=REMOVE_SEEN)\n", - " for metric in RANKING_METRICS:\n", - " test_results[metric] = eval(metric)(test, all_predictions, col_prediction='prediction', k=K)\n", - " return test_results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_results_tpe = compute_test_results(svd)\n", - "print(test_results_tpe)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6. More Tuning Algorithms\n", - "We now apply other tuning algorithms supported by NNI to the same problem. For details about these tuners, see the [NNI docs.](https://nni.readthedocs.io/en/latest/tuners.html#)\n", - "The only change needed is in the relevant entry in the configuration file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In summary, the tuners used in this notebook are the following:\n", - "- Tree-structured Parzen Estimator (TPE), within the Sequential Model-Based Optimization (SMBO) framework,\n", - "- SMAC, also an instance of SMBO,\n", - "- Hyperband\n", - "- Metis, an implementation of Bayesian optimization with Gaussian Processes\n", - "- a Naive Evolutionary algorithm\n", - "- an Annealing method for sampling, and \n", - "- plain Random Search as a baseline. \n", - "\n", - "For more details and references to the relevant literature, see the [NNI github](https://github.com/Microsoft/nni/blob/master/docs/en_US/Builtin_Tuner.md)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Random search\n", - "config['tuner']['builtinTunerName'] = 'Random'\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))\n", - " print(config_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stop_nni()\n", - "with Timer() as time_random:\n", - " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 1.0153957155486038,\n", + " 'ndcg_at_k': 0.027899886410413542,\n", + " 'precision_at_k': 0.024708377518557794}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "test_results_random = compute_test_results(svd)" + "best_metrics_svd" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'parameter_id': 22,\n", + " 'parameter_source': 'algorithm',\n", + " 'parameters': {'n_factors': 100,\n", + " 'init_mean': 0.11174254678268791,\n", + " 'init_std_dev': 0.1977986447321912,\n", + " 'lr_bu': 0.008056394852767297,\n", + " 'lr_bi': 0.0008200509033991312,\n", + " 'lr_pu': 0.006790668636338857,\n", + " 'lr_qi': 0.09214174394023733,\n", + " 'reg_bu': 0.7866760462023896,\n", + " 'reg_bi': 0.23022204903863294,\n", + " 'reg_pu': 0.9432527716016627,\n", + " 'reg_qi': 0.9808218483424879},\n", + " 'parameter_index': 0}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Annealing\n", - "config['tuner']['builtinTunerName'] = 'Anneal'\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" + "best_params_svd" ] }, { @@ -756,8 +601,9 @@ "outputs": [], "source": [ "stop_nni()\n", - "with Timer() as time_anneal:\n", - " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "config_path_ncf = os.path.join(TMP_DIR, 'config_ncf.yml')\n", + "with Timer() as time_ncf:\n", + " start_nni(config_path_ncf, wait=WAITING_TIME, max_retries=MAX_RETRIES)" ] }, { @@ -767,20 +613,7 @@ "outputs": [], "source": [ "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "test_results_anneal = compute_test_results(svd)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Naive evolutionary search\n", - "config['tuner']['builtinTunerName'] = 'Evolution'\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" + "trials_ncf, best_metrics_ncf, best_params_ncf, best_trial_path_ncf = get_trials('maximize')" ] }, { @@ -789,9 +622,7 @@ "metadata": {}, "outputs": [], "source": [ - "stop_nni()\n", - "with Timer() as time_evolution:\n", - " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "best_metrics_ncf" ] }, { @@ -800,92 +631,16 @@ "metadata": {}, "outputs": [], "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "test_results_evolution = compute_test_results(svd)" + "best_params_ncf" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The SMAC tuner requires to have been installed with the following command
\n", - "`nnictl package install --name=SMAC`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# SMAC\n", - "config['tuner']['builtinTunerName'] = 'SMAC'\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check if installed\n", - "proc = subprocess.run([sys.prefix + '/bin/nnictl', 'package', 'show'], stdout=subprocess.PIPE)\n", - "if proc.returncode != 0:\n", - " raise RuntimeError(\"'nnictl package show' failed with code %d\" % proc.returncode)\n", - "if 'SMAC' not in proc.stdout.decode().strip().split():\n", - " proc = subprocess.run([sys.prefix + '/bin/nnictl', 'package', 'install', '--name=SMAC'])\n", - " if proc.returncode != 0:\n", - " raise RuntimeError(\"'nnictl package install' failed with code %d\" % proc.returncode)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Skipping SMAC optimization for now\n", - "# stop_nni()\n", - "with Timer() as time_smac:\n", - "# start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#check_metrics_written()\n", - "#svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "#test_results_smac = compute_test_results(svd)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Metis\n", - "config['tuner']['builtinTunerName'] = 'MetisTuner'\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stop_nni()\n", - "with Timer() as time_metis:\n", - " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "### 5. Show Results\n", + "\n", + "The metrics for each model type is reported on the validation set. At this point we can compare the metrics for each model and select the one with the best score on the primary metric(s) of interest." ] }, { @@ -894,16 +649,22 @@ "metadata": {}, "outputs": [], "source": [ - "check_metrics_written()\n", - "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "test_results_metis = compute_test_results(svd)" + "def combine_metrics_dicts(*metrics):\n", + " df = pd.DataFrame(metrics[0], index=[0])\n", + " for metric in metrics[1:]:\n", + " df = df.append(pd.DataFrame(metric, index=[0]))\n", + " return df\n", + "\n", + "best_metrics_svd['name'] = 'svd'\n", + "best_metrics_ncf['name'] = 'ncf'\n", + "combine_metrics_dicts(best_metrics_svd, best_metrics_ncf)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Hyperband follows a different style of configuration from other tuners. See [the NNI documentation](https://nni.readthedocs.io/en/latest/hyperbandAdvisor.html). Note that the [training script](../../reco_utils/nni/svd_training.py) needs to be adjusted as well, since each Hyperband trial receives an additional parameter `STEPS`, which corresponds to the resource allocation _ri_ in the [Hyperband algorithm](https://arxiv.org/pdf/1603.06560.pdf). In this example, we used `STEPS` in combination with `R` to determine the number of epochs that SVD will run for in every trial." + "Once we select our model based on the validation set, we can test the model's performance on the test set using the best hyperparameters for the best model (in this case we will simply choose the NCF model, your results may differ depending on your own tests). We will do so by retraining the model on both the train and validation sets to predict on the test set" ] }, { @@ -912,18 +673,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Hyperband\n", - "config['advisor'] = {\n", - " 'builtinAdvisorName': 'Hyperband',\n", - " 'classArgs': {\n", - " 'R': NUM_EPOCHS,\n", - " 'eta': 3,\n", - " 'optimize_mode': 'maximize'\n", - " }\n", - "}\n", - "config.pop('tuner')\n", - "with open(config_path, 'w') as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" + "train_and_validation = train.append(validation).reset_index(drop=True)\n", + "data = NCFDataset(train_and_validation, test, seed=DEFAULT_SEED)" ] }, { @@ -932,9 +683,16 @@ "metadata": {}, "outputs": [], "source": [ - "stop_nni()\n", - "with Timer() as time_hyperband:\n", - " start_nni(config_path, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "model = NCF(\n", + " n_users=data.n_users, \n", + " n_items=data.n_items,\n", + " model_type=\"NeuMF\",\n", + " n_factors=best_params_ncf[\"parameters\"][\"n_factors\"],\n", + " n_epochs=NUM_EPOCHS,\n", + " learning_rate=best_params_ncf[\"parameters\"][\"learning_rate\"],\n", + " verbose=True,\n", + " seed=DEFAULT_SEED\n", + ")" ] }, { @@ -943,9 +701,7 @@ "metadata": {}, "outputs": [], "source": [ - "check_metrics_written()\n", - "svd = surprise.dump.load(os.path.join(get_trials('maximize')[3], \"model.dump\"))[1]\n", - "test_results_hyperband = compute_test_results(svd)" + "model.fit(data)" ] }, { @@ -954,13 +710,37 @@ "metadata": {}, "outputs": [], "source": [ - "test_results_tpe.update({'time': time_tpe.interval})\n", - "test_results_random.update({'time': time_random.interval})\n", - "test_results_anneal.update({'time': time_anneal.interval})\n", - "test_results_evolution.update({'time': time_evolution.interval})\n", - "#test_results_smac.update({'time': time_smac.interval})\n", - "test_results_metis.update({'time': time_metis.interval})\n", - "test_results_hyperband.update({'time': time_hyperband.interval})" + "def compute_test_results(model, train, test):\n", + " test_results = {}\n", + " \n", + " # Rating Metrics\n", + " predictions = [[row.userID, row.itemID, model.predict(row.userID, row.itemID)]\n", + " for (_, row) in test.iterrows()]\n", + "\n", + " predictions = pd.DataFrame(predictions, columns=['userID', 'itemID', 'prediction'])\n", + " predictions = predictions.astype({'userID': 'int64', 'itemID': 'int64', 'prediction': 'float64'})\n", + "\n", + " for metric in RATING_METRICS:\n", + " test_results[metric] = eval(metric)(test, predictions)\n", + " \n", + " # Ranking Metrics\n", + " users, items, preds = [], [], []\n", + " item = list(train.itemID.unique())\n", + " for user in train.userID.unique():\n", + " user = [user] * len(item) \n", + " users.extend(user)\n", + " items.extend(item)\n", + " preds.extend(list(model.predict(user, item, is_list=True)))\n", + "\n", + " all_predictions = pd.DataFrame(data={\"userID\": users, \"itemID\": items, \"prediction\": preds})\n", + "\n", + " merged = pd.merge(train, all_predictions, on=[\"userID\", \"itemID\"], how=\"outer\")\n", + " all_predictions = merged[merged.rating.isnull()].drop('rating', axis=1)\n", + "\n", + " for metric in RANKING_METRICS:\n", + " test_results[metric] = eval(metric)(test, all_predictions, col_prediction='prediction', k=K)\n", + " \n", + " return test_results" ] }, { @@ -969,33 +749,8 @@ "metadata": {}, "outputs": [], "source": [ - "algos = [\"TPE\", \n", - " \"Random Search\", \n", - " \"Annealing\", \n", - " \"Evolution\", \n", - " #\"SMAC\", \n", - " \"Metis\", \n", - " \"Hyperband\"]\n", - "res_df = pd.DataFrame(index=algos,\n", - " data=[res for res in [test_results_tpe, \n", - " test_results_random, \n", - " test_results_anneal, \n", - " test_results_evolution, \n", - " #test_results_smac, \n", - " test_results_metis, \n", - " test_results_hyperband]] \n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "res_df.sort_values(by=\"precision_at_k\", ascending=False).round(3)" + "test_results = compute_test_results(model, train_and_validation, test)\n", + "print(test_results)" ] }, { @@ -1032,8 +787,12 @@ "source": [ "### 7. Concluding Remarks\n", "\n", - "We showed how to tune **all** the hyperparameters accepted by Surprise SVD simultaneously, by utilizing the NNI toolkit. \n", - "For example, training and evaluation of a single SVD model takes about 50 seconds on the 100k MovieLens data on a Standard D2_V2 VM. Searching through 100 different combinations of hyperparameters sequentially would take about 80 minutes whereas each of the above experiments took about 10 minutes by exploiting parallelization on a single D16_v3 VM. With NNI, one can take advantage of concurrency and multiple processors on a virtual machine and can use a variety of tuning methods to navigate efficiently through a large space of hyperparameters.
\n", + "In this notebook we showed how to use the NNI framework on different models. By inspection of the training scripts, the differences between the two should help you identify what components would need to be modified to run another model with NNI.\n", + "\n", + "In practice, an AutoML framework like NNI is just a tool to help you explore a large space of hyperparameters quickly with a pre-described level of randomization. It is recommended that in addition to using NNI one trains baseline models using typical hyperparamter choices (learning rate of 0.005, 0.001 or regularization rates of 0.05, 0.01, etc.) to draw more meaningful comparisons between model performances. This may help determine if a model is overfitting from the tuner or if there is a statistically significant improvement.\n", + "\n", + "Another thing to note is the added computational cost required to train models using an AutoML framework. In this case, it takes about 1 minute to train each of the models on a [Standard_NC6 VM](https://docs.microsoft.com/en-us/azure/virtual-machines/nc-series). With this in mind, while NNI can easily train hundreds of models over all hyperparameters for a model, in practice it may be beneficial to choose a subset of the hyperparameters that are deemed most important and to tune those. Too small of a hyperparameter search space may restrict our exploration, but too large may also lead to random noise in the data being exploited by a specific combination of hyperparameters. \n", + "\n", "For examples of scaling larger tuning workloads on clusters of machines, see [the notebooks](./README.md) that employ the [Azure Machine Learning service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters). " ] }, @@ -1041,11 +800,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### References\n", + "### 8. References\n", + "\n", + "Recommenders Repo References\n", + "* [SVD deep-dive notebook](../02_model/surprise_svd_deep_dive.ipynb)\n", + "* [NCF deep-dive notebook](../02_model/ncf_deep_dive.ipynb)\n", + "* [SVD + NNI model optimization](./nni_surprise_svd.ipynb)\n", "\n", - "* [Matrix factorization algorithms in Surprise](https://surprise.readthedocs.io/en/stable/matrix_factorization.html) \n", - "* [Surprise SVD deep-dive notebook](../02_model/surprise_svd_deep_dive.ipynb)\n", - "* [Neural Network Intelligence toolkit](https://github.com/Microsoft/nni)" + "External References\n", + "* [Surprise Docs | Matrix factorization algorithms](https://surprise.readthedocs.io/en/stable/matrix_factorization.html) \n", + "* [NNI Docs | Neural Network Intelligence toolkit](https://github.com/Microsoft/nni)" ] } ], From 70d5420f1e1d50e04ac0b87e83d6d77c64b3942d Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 23 Apr 2020 06:30:41 +0000 Subject: [PATCH 07/17] clean up ncf training --- reco_utils/tuning/nni/ncf_training.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/reco_utils/tuning/nni/ncf_training.py b/reco_utils/tuning/nni/ncf_training.py index e00d87967a..4bd9449f0d 100644 --- a/reco_utils/tuning/nni/ncf_training.py +++ b/reco_utils/tuning/nni/ncf_training.py @@ -8,17 +8,10 @@ import os import pandas as pd import nni -import surprise import sys sys.path.append("../../../") -import reco_utils.evaluation.python_evaluation as evaluation -from reco_utils.recommender.surprise.surprise_utils import ( - predict, - compute_ranking_predictions, -) - from reco_utils.recommender.ncf.ncf_singlenode import NCF from reco_utils.recommender.ncf.dataset import Dataset as NCFDataset from reco_utils.dataset import movielens From 75c2999e05aac90d34ab4c181d3b873b81a6ab4d Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 23 Apr 2020 06:31:17 +0000 Subject: [PATCH 08/17] clean up ncf markdown --- .../nni_ncf.ipynb | 124 +++++++++++++++--- 1 file changed, 108 insertions(+), 16 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index a8a9fc6936..c728f1426b 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -596,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -608,7 +608,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -618,18 +618,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 3.2150175063137225,\n", + " 'ndcg_at_k': 0.08656122568252186,\n", + " 'precision_at_k': 0.07433722163308593}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_metrics_ncf" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'parameter_id': 20,\n", + " 'parameter_source': 'algorithm',\n", + " 'parameters': {'n_factors': 12, 'learning_rate': 0.03713547464301225},\n", + " 'parameter_index': 0}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_params_ncf" ] @@ -645,9 +672,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rmsendcg_at_kprecision_at_kname
01.0153960.0279000.024708svd
03.2150180.0865610.074337ncf
\n", + "
" + ], + "text/plain": [ + " rmse ndcg_at_k precision_at_k name\n", + "0 1.015396 0.027900 0.024708 svd\n", + "0 3.215018 0.086561 0.074337 ncf" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def combine_metrics_dicts(*metrics):\n", " df = pd.DataFrame(metrics[0], index=[0])\n", @@ -669,7 +753,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -679,7 +763,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -697,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -706,7 +790,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -745,9 +829,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'rmse': 3.2183926691724394, 'precision_at_k': 0.02682926829268293, 'ndcg_at_k': 0.0297817565086106}\n" + ] + } + ], "source": [ "test_results = compute_test_results(model, train_and_validation, test)\n", "print(test_results)" @@ -764,7 +856,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -774,7 +866,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ From d71ab3f6916f58ba984d86b7c09b4668864e68dd Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 23 Apr 2020 06:38:16 +0000 Subject: [PATCH 09/17] removed unused params in ncf training --- reco_utils/tuning/nni/ncf_training.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/reco_utils/tuning/nni/ncf_training.py b/reco_utils/tuning/nni/ncf_training.py index 4bd9449f0d..45fedbcce0 100644 --- a/reco_utils/tuning/nni/ncf_training.py +++ b/reco_utils/tuning/nni/ncf_training.py @@ -38,17 +38,6 @@ def ncf_training(params): data = NCFDataset(train=train_data, test=validation_data, seed=DEFAULT_SEED) - ncf_params = { - p: params[p] - for p in [ - "random_state", - "n_epochs", - "verbose", - "n_factors", - "learning_rate", - ] - } - model = NCF ( n_users=data.n_users, n_items=data.n_items, From 77214efd9d44fa5a033da0d03f5249e050baf9c2 Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 23 Apr 2020 06:45:55 +0000 Subject: [PATCH 10/17] update readme --- notebooks/04_model_select_and_optimize/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/04_model_select_and_optimize/README.md b/notebooks/04_model_select_and_optimize/README.md index 11374d6927..c78f59db1f 100644 --- a/notebooks/04_model_select_and_optimize/README.md +++ b/notebooks/04_model_select_and_optimize/README.md @@ -8,6 +8,7 @@ In this directory, notebooks are provided to demonstrate how to tune and optimiz | [azureml_hyperdrive_wide_and_deep](azureml_hyperdrive_wide_and_deep.ipynb) | Quickstart tutorial on utilizing [Azure Machine Learning service](https://azure.microsoft.com/en-us/services/machine-learning-service/) for hyperparameter tuning of wide-and-deep model. | [azureml_hyperdrive_surprise_svd](azureml_hyperdrive_surprise_svd.ipynb) | Quickstart tutorial on utilizing [Azure Machine Learning service](https://azure.microsoft.com/en-us/services/machine-learning-service/) for hyperparameter tuning of the matrix factorization method SVD from [Surprise library](https://surprise.readthedocs.io/en/stable/). | [nni_surprise_svd](nni_surprise_svd.ipynb) | Quickstart tutorial on utilizing the [Neural Network Intelligence toolkit](https://github.com/Microsoft/nni) for hyperparameter tuning of the matrix factorization method SVD from [Surprise library](https://surprise.readthedocs.io/en/stable/). +| [nni_ncf](nni_ncf.ipynb) | Quickstart tutorial on utilizing the [Neural Network Intelligence toolkit](https://github.com/Microsoft/nni) as a tool to tune the [NCF model](../02_model/ncf_deep_dive.ipynb) and [SVD model](../02_model/surprise_svd_deep_dive.ipynb) and compare their performance against one another ### Prerequisites To run the examples running on the Azure Machine Learning service, the [`azureml-sdk`](https://pypi.org/project/azureml-sdk/) is required. The AzureML Python SDK is already installed after setting up the conda environments from this repository (see [SETUP.md](../../SETUP.md)). From 4cb36ae8b61cf89f9090f3f718e7ce784d0a2b74 Mon Sep 17 00:00:00 2001 From: seanytak Date: Mon, 27 Apr 2020 07:01:22 +0000 Subject: [PATCH 11/17] remove svd components --- .../nni_ncf.ipynb | 304 +++++------------- 1 file changed, 83 insertions(+), 221 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index c728f1426b..a9526aed52 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -14,15 +14,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters for the Neural Collaborative Filtering Model and Surprise SVD model.\n", + "This notebook shows how to use the **[Neural Network Intelligence](https://nni.readthedocs.io/en/latest/) toolkit (NNI)** for tuning hyperparameters for the Neural Collaborative Filtering Model.\n", "\n", - "To learn about each tuner NNI offers you can read about it [here](https://nni.readthedocs.io/en/latest/Tuner/BuiltinTuner.html). To see how each tuner performs on the Surprise SVD model, visit [this notebook instead](./nni_surprise_svd.ipynb). \n", + "To learn about each tuner NNI offers you can read about it [here](https://nni.readthedocs.io/en/latest/Tuner/BuiltinTuner.html).\n", "\n", - "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. . NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", + "NNI is a toolkit to help users design and tune machine learning models (e.g., hyperparameters), neural network architectures, or complex system’s parameters, in an efficient and automatic way. NNI has several appealing properties: ease of use, scalability, flexibility and efficiency. NNI can be executed in a distributed way on a local machine, a remote server, or a large scale training platform such as OpenPAI or Kubernetes. \n", "\n", "In this notebook, we can see how NNI works with two different model types and the differences between their hyperparameter search spaces, yaml config file, and training scripts.\n", "\n", - "- [Surprise SVD Training Script](../../reco_utils/nni/svd_training.py)\n", "- [NCF Training Script](../../reco_utils/nni/ncf_training.py)\n", "\n", "In all experiments, we maximize precision@10. \n", @@ -349,83 +348,6 @@ " script_params += \" --remove-seen\"" ] }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "# hyperparameters search space\n", - "# We do not set 'lr_all' and 'reg_all' because they will be overriden by the other lr_ and reg_ parameters\n", - "\n", - "svd_hyper_params = {\n", - " 'n_factors': {\"_type\": \"choice\", \"_value\": [10, 50, 100, 150, 200]},\n", - " 'init_mean': {\"_type\": \"uniform\", \"_value\": [-0.5, 0.5]},\n", - " 'init_std_dev': {\"_type\": \"uniform\", \"_value\": [0.01, 0.2]},\n", - " 'lr_bu': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", - " 'lr_bi': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", - " 'lr_pu': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", - " 'lr_qi': {\"_type\": \"uniform\", \"_value\": [1e-6, 0.1]}, \n", - " 'reg_bu': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]},\n", - " 'reg_bi': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}, \n", - " 'reg_pu': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}, \n", - " 'reg_qi': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]}\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "with open(os.path.join(TMP_DIR, 'search_space_svd.json'), 'w') as fp:\n", - " json.dump(svd_hyper_params, fp)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We also create a yaml file for the configuration of the trials and the tuning algorithm to be used (in this experiment we use the [TPE tuner](https://nni.readthedocs.io/en/latest/hyperoptTuner.html)). " - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "config = {\n", - " \"authorName\": \"default\",\n", - " \"experimentName\": \"surprise_svd\",\n", - " \"trialConcurrency\": 8,\n", - " \"maxExecDuration\": \"1h\",\n", - " \"maxTrialNum\": MAX_TRIAL_NUM,\n", - " \"trainingServicePlatform\": \"local\",\n", - " # The path to Search Space\n", - " \"searchSpacePath\": \"search_space_svd.json\",\n", - " \"useAnnotation\": False,\n", - " \"logDir\": LOG_DIR,\n", - " \"tuner\": {\n", - " \"builtinTunerName\": \"TPE\",\n", - " \"classArgs\": {\n", - " #choice: maximize, minimize\n", - " \"optimize_mode\": \"maximize\"\n", - " }\n", - " },\n", - " # The path and the running command of trial\n", - " \"trial\": {\n", - " \"command\": sys.prefix + \"/bin/python svd_training.py\" + \" \" + script_params,\n", - " \"codeDir\": os.path.join(os.path.split(os.path.abspath(reco_utils.__file__))[0], \"tuning\", \"nni\"),\n", - " \"gpuNum\": 0\n", - " }\n", - "}\n", - " \n", - "with open(os.path.join(TMP_DIR, \"config_svd.yml\"), \"w\") as fp:\n", - " fp.write(yaml.dump(config, default_flow_style=False))" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -519,146 +441,154 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "stop_nni()\n", - "config_path_svd = os.path.join(TMP_DIR, 'config_svd.yml')\n", - "with Timer() as time_svd:\n", - " start_nni(config_path_svd, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "config_path_ncf = os.path.join(TMP_DIR, 'config_ncf.yml')\n", + "with Timer() as time_ncf:\n", + " start_nni(config_path_ncf, wait=WAITING_TIME, max_retries=MAX_RETRIES)" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "trials_svd, best_metrics_svd, best_params_svd, best_trial_path_svd = get_trials('maximize')" + "trials_ncf, best_metrics_ncf, best_params_ncf, best_trial_path_ncf = get_trials('maximize')" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'rmse': 1.0153957155486038,\n", - " 'ndcg_at_k': 0.027899886410413542,\n", - " 'precision_at_k': 0.024708377518557794}" + "{'rmse': 3.2150175063137225,\n", + " 'ndcg_at_k': 0.08656122568252186,\n", + " 'precision_at_k': 0.07433722163308593}" ] }, - "execution_count": 15, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "best_metrics_svd" + "best_metrics_ncf" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'parameter_id': 22,\n", + "{'parameter_id': 20,\n", " 'parameter_source': 'algorithm',\n", - " 'parameters': {'n_factors': 100,\n", - " 'init_mean': 0.11174254678268791,\n", - " 'init_std_dev': 0.1977986447321912,\n", - " 'lr_bu': 0.008056394852767297,\n", - " 'lr_bi': 0.0008200509033991312,\n", - " 'lr_pu': 0.006790668636338857,\n", - " 'lr_qi': 0.09214174394023733,\n", - " 'reg_bu': 0.7866760462023896,\n", - " 'reg_bi': 0.23022204903863294,\n", - " 'reg_pu': 0.9432527716016627,\n", - " 'reg_qi': 0.9808218483424879},\n", + " 'parameters': {'n_factors': 12, 'learning_rate': 0.03713547464301225},\n", " 'parameter_index': 0}" ] }, - "execution_count": 16, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "best_params_svd" + "best_params_ncf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Baseline Model\n", + "\n", + "Although we hope that the additional effort of utilizing an AutoML framework like NNI for hyperparameter tuning will lead to better results, we should also draw comparisons using our baseline model (our model trained with its default hyperparameters). This allows us to precisely understand what performance benefits NNI is or isn't providing." ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "stop_nni()\n", - "config_path_ncf = os.path.join(TMP_DIR, 'config_ncf.yml')\n", - "with Timer() as time_ncf:\n", - " start_nni(config_path_ncf, wait=WAITING_TIME, max_retries=MAX_RETRIES)" + "train_and_validation = train.append(validation).reset_index(drop=True)\n", + "data = NCFDataset(train_and_validation, test, seed=DEFAULT_SEED)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", - "trials_ncf, best_metrics_ncf, best_params_ncf, best_trial_path_ncf = get_trials('maximize')" + "model = NCF(\n", + " n_users=data.n_users, \n", + " n_items=data.n_items,\n", + " model_type=\"NeuMF\",\n", + " n_epochs=NUM_EPOCHS, \n", + " verbose=True,\n", + " seed=DEFAULT_SEED\n", + ")\n", + "model.fit(data)" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rmse': 3.2150175063137225,\n", - " 'ndcg_at_k': 0.08656122568252186,\n", - " 'precision_at_k': 0.07433722163308593}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "best_metrics_ncf" + "def compute_test_results(model, train, test):\n", + " test_results = {}\n", + " \n", + " # Rating Metrics\n", + " predictions = [[row.userID, row.itemID, model.predict(row.userID, row.itemID)]\n", + " for (_, row) in test.iterrows()]\n", + "\n", + " predictions = pd.DataFrame(predictions, columns=['userID', 'itemID', 'prediction'])\n", + " predictions = predictions.astype({'userID': 'int64', 'itemID': 'int64', 'prediction': 'float64'})\n", + "\n", + " for metric in RATING_METRICS:\n", + " test_results[metric] = eval(metric)(test, predictions)\n", + " \n", + " # Ranking Metrics\n", + " users, items, preds = [], [], []\n", + " item = list(train.itemID.unique())\n", + " for user in train.userID.unique():\n", + " user = [user] * len(item) \n", + " users.extend(user)\n", + " items.extend(item)\n", + " preds.extend(list(model.predict(user, item, is_list=True)))\n", + "\n", + " all_predictions = pd.DataFrame(data={\"userID\": users, \"itemID\": items, \"prediction\": preds})\n", + "\n", + " merged = pd.merge(train, all_predictions, on=[\"userID\", \"itemID\"], how=\"outer\")\n", + " all_predictions = merged[merged.rating.isnull()].drop('rating', axis=1)\n", + "\n", + " for metric in RANKING_METRICS:\n", + " test_results[metric] = eval(metric)(test, all_predictions, col_prediction='prediction', k=K)\n", + " \n", + " return test_results" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'parameter_id': 20,\n", - " 'parameter_source': 'algorithm',\n", - " 'parameters': {'n_factors': 12, 'learning_rate': 0.03713547464301225},\n", - " 'parameter_index': 0}" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "best_params_ncf" + "test_results = compute_test_results(model, train_and_validation, test)\n", + "print(test_results)" ] }, { @@ -739,9 +669,9 @@ " df = df.append(pd.DataFrame(metric, index=[0]))\n", " return df\n", "\n", - "best_metrics_svd['name'] = 'svd'\n", - "best_metrics_ncf['name'] = 'ncf'\n", - "combine_metrics_dicts(best_metrics_svd, best_metrics_ncf)" + "test_results['name'] =''\n", + "best_metrics_ncf['name'] = 'ncf_tuned'\n", + "combine_metrics_dicts(test_results, best_metrics_ncf)" ] }, { @@ -751,16 +681,6 @@ "Once we select our model based on the validation set, we can test the model's performance on the test set using the best hyperparameters for the best model (in this case we will simply choose the NCF model, your results may differ depending on your own tests). We will do so by retraining the model on both the train and validation sets to predict on the test set" ] }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [], - "source": [ - "train_and_validation = train.append(validation).reset_index(drop=True)\n", - "data = NCFDataset(train_and_validation, test, seed=DEFAULT_SEED)" - ] - }, { "cell_type": "code", "execution_count": 23, @@ -788,63 +708,6 @@ "model.fit(data)" ] }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "def compute_test_results(model, train, test):\n", - " test_results = {}\n", - " \n", - " # Rating Metrics\n", - " predictions = [[row.userID, row.itemID, model.predict(row.userID, row.itemID)]\n", - " for (_, row) in test.iterrows()]\n", - "\n", - " predictions = pd.DataFrame(predictions, columns=['userID', 'itemID', 'prediction'])\n", - " predictions = predictions.astype({'userID': 'int64', 'itemID': 'int64', 'prediction': 'float64'})\n", - "\n", - " for metric in RATING_METRICS:\n", - " test_results[metric] = eval(metric)(test, predictions)\n", - " \n", - " # Ranking Metrics\n", - " users, items, preds = [], [], []\n", - " item = list(train.itemID.unique())\n", - " for user in train.userID.unique():\n", - " user = [user] * len(item) \n", - " users.extend(user)\n", - " items.extend(item)\n", - " preds.extend(list(model.predict(user, item, is_list=True)))\n", - "\n", - " all_predictions = pd.DataFrame(data={\"userID\": users, \"itemID\": items, \"prediction\": preds})\n", - "\n", - " merged = pd.merge(train, all_predictions, on=[\"userID\", \"itemID\"], how=\"outer\")\n", - " all_predictions = merged[merged.rating.isnull()].drop('rating', axis=1)\n", - "\n", - " for metric in RANKING_METRICS:\n", - " test_results[metric] = eval(metric)(test, all_predictions, col_prediction='prediction', k=K)\n", - " \n", - " return test_results" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'rmse': 3.2183926691724394, 'precision_at_k': 0.02682926829268293, 'ndcg_at_k': 0.0297817565086106}\n" - ] - } - ], - "source": [ - "test_results = compute_test_results(model, train_and_validation, test)\n", - "print(test_results)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -895,12 +758,11 @@ "### 8. References\n", "\n", "Recommenders Repo References\n", - "* [SVD deep-dive notebook](../02_model/surprise_svd_deep_dive.ipynb)\n", "* [NCF deep-dive notebook](../02_model/ncf_deep_dive.ipynb)\n", "* [SVD + NNI model optimization](./nni_surprise_svd.ipynb)\n", "\n", "External References\n", - "* [Surprise Docs | Matrix factorization algorithms](https://surprise.readthedocs.io/en/stable/matrix_factorization.html) \n", + "* [NCF Paper](https://arxiv.org/abs/1708.05031) \n", "* [NNI Docs | Neural Network Intelligence toolkit](https://github.com/Microsoft/nni)" ] } From 4f491fb4341628bab06f98c753ddb0a7c469e0cc Mon Sep 17 00:00:00 2001 From: seanytak Date: Mon, 27 Apr 2020 07:39:01 +0000 Subject: [PATCH 12/17] fix up final test results reporting --- .../nni_ncf.ipynb | 162 ++++-------------- 1 file changed, 31 insertions(+), 131 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index a9526aed52..699c63ffde 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -24,8 +24,6 @@ "\n", "- [NCF Training Script](../../reco_utils/nni/ncf_training.py)\n", "\n", - "In all experiments, we maximize precision@10. \n", - "\n", "For this notebook we use a _local machine_ as the training platform (this can be any machine running the `reco_base` conda environment). In this case, NNI uses the available processors of the machine to parallelize the trials, subject to the value of `trialConcurrency` we specify in the configuration. Our runs and the results we report were obtained on a [Standard_D16_v3 virtual machine](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/sizes-general#dv3-series-1) with 16 vcpus and 64 GB memory." ] }, @@ -149,7 +147,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.0kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.8kKB/s]\n" ] }, { @@ -357,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -369,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -381,15 +379,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Our NCF config.yml file follows the same structure as the SVD config.yml. The only differences are the\n", - "- Experiment name\n", - "- Hyperparameter Search Space\n", - "- The executed command for the trial (Note: The script parameters have been configured to work for both training scripts)" + "This config file follows the guidelines provided in [NNI Experiment Config instructions](https://github.com/microsoft/nni/blob/master/docs/en_US/Tutorial/ExperimentConfig.md).\n", + "\n", + "The options to pay attention to are\n", + "- The \"searchSpacePath\" which contains the space of hyperparameters we wanted to tune defined above\n", + "- The \"tuner\" which specifies the hyperparameter tuning algorithm that will sample from our search space and optimize our model" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -441,7 +440,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -453,7 +452,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -463,45 +462,18 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rmse': 3.2150175063137225,\n", - " 'ndcg_at_k': 0.08656122568252186,\n", - " 'precision_at_k': 0.07433722163308593}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_metrics_ncf" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'parameter_id': 20,\n", - " 'parameter_source': 'algorithm',\n", - " 'parameters': {'n_factors': 12, 'learning_rate': 0.03713547464301225},\n", - " 'parameter_index': 0}" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "best_params_ncf" ] @@ -521,16 +493,7 @@ "metadata": {}, "outputs": [], "source": [ - "train_and_validation = train.append(validation).reset_index(drop=True)\n", - "data = NCFDataset(train_and_validation, test, seed=DEFAULT_SEED)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "data = NCFDataset(train, validation, seed=DEFAULT_SEED)\n", "model = NCF(\n", " n_users=data.n_users, \n", " n_items=data.n_items,\n", @@ -587,8 +550,8 @@ "metadata": {}, "outputs": [], "source": [ - "test_results = compute_test_results(model, train_and_validation, test)\n", - "print(test_results)" + "test_results = compute_test_results(model, train, validation)\n", + "test_results" ] }, { @@ -602,66 +565,9 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
rmsendcg_at_kprecision_at_kname
01.0153960.0279000.024708svd
03.2150180.0865610.074337ncf
\n", - "
" - ], - "text/plain": [ - " rmse ndcg_at_k precision_at_k name\n", - "0 1.015396 0.027900 0.024708 svd\n", - "0 3.215018 0.086561 0.074337 ncf" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "def combine_metrics_dicts(*metrics):\n", " df = pd.DataFrame(metrics[0], index=[0])\n", @@ -669,7 +575,7 @@ " df = df.append(pd.DataFrame(metric, index=[0]))\n", " return df\n", "\n", - "test_results['name'] =''\n", + "test_results['name'] = 'ncf_baseline'\n", "best_metrics_ncf['name'] = 'ncf_tuned'\n", "combine_metrics_dicts(test_results, best_metrics_ncf)" ] @@ -683,10 +589,11 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ + "train_and_validation = train.append(validation).reset_index(drop=True)\n", "model = NCF(\n", " n_users=data.n_users, \n", " n_items=data.n_items,\n", @@ -696,30 +603,23 @@ " learning_rate=best_params_ncf[\"parameters\"][\"learning_rate\"],\n", " verbose=True,\n", " seed=DEFAULT_SEED\n", - ")" + ")\n", + "model.fit(data)" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "model.fit(data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As we see in the table above, _annealing_ performs best with respect to the primary metric (precision@10) that all the tuners optimized. Also the best NDCG@10 is obtained for annealing and correlates well with precision@10. RMSE on the other hand does not correlate well and is not optimized for annealing, since finding the top k recommendations in the right order is a different task from predicting ratings (high and low) accurately. \n", - "We have also observed that the above ranking of the tuners is not consistent and may change when trying these experiments multiple times. Since some of these tuners rely heavily on randomized sampling, a larger number of trials is required to get more consistent metrics.\n", - "In addition, some of the tuning algorithms themselves come with parameters, which can affect their performance." + "results = compute_test_results(model, train_and_validation, test)\n", + "results" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -729,7 +629,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -759,7 +659,7 @@ "\n", "Recommenders Repo References\n", "* [NCF deep-dive notebook](../02_model/ncf_deep_dive.ipynb)\n", - "* [SVD + NNI model optimization](./nni_surprise_svd.ipynb)\n", + "* [SVD NNI notebook (uses more tuners available)](./nni_surprise_svd.ipynb)\n", "\n", "External References\n", "* [NCF Paper](https://arxiv.org/abs/1708.05031) \n", From ead671c6efec24cbcb875afd6510e0705cbfcf8f Mon Sep 17 00:00:00 2001 From: seanytak Date: Wed, 29 Apr 2020 00:26:42 +0000 Subject: [PATCH 13/17] remove output from cell with warnings --- notebooks/02_model/ncf_deep_dive.ipynb | 195 ++++--------------------- 1 file changed, 29 insertions(+), 166 deletions(-) diff --git a/notebooks/02_model/ncf_deep_dive.ipynb b/notebooks/02_model/ncf_deep_dive.ipynb index 046c42563f..b978b79505 100644 --- a/notebooks/02_model/ncf_deep_dive.ipynb +++ b/notebooks/02_model/ncf_deep_dive.ipynb @@ -224,7 +224,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.7kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.6kKB/s]\n" ] }, { @@ -398,7 +398,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 806.4961465689998 seconds for training.\n" + "Took 663.2377220259996 seconds for training.\n" ] } ], @@ -461,31 +461,31 @@ " 0\n", " 1.0\n", " 149.0\n", - " 0.047002\n", + " 0.029076\n", " \n", " \n", " 1\n", " 1.0\n", " 88.0\n", - " 0.566155\n", + " 0.632505\n", " \n", " \n", " 2\n", " 1.0\n", " 101.0\n", - " 0.226456\n", + " 0.492521\n", " \n", " \n", " 3\n", " 1.0\n", " 110.0\n", - " 0.008283\n", + " 0.081617\n", " \n", " \n", " 4\n", " 1.0\n", " 103.0\n", - " 0.005900\n", + " 0.004513\n", " \n", " \n", "\n", @@ -493,11 +493,11 @@ ], "text/plain": [ " userID itemID prediction\n", - "0 1.0 149.0 0.047002\n", - "1 1.0 88.0 0.566155\n", - "2 1.0 101.0 0.226456\n", - "3 1.0 110.0 0.008283\n", - "4 1.0 103.0 0.005900" + "0 1.0 149.0 0.029076\n", + "1 1.0 88.0 0.632505\n", + "2 1.0 101.0 0.492521\n", + "3 1.0 110.0 0.081617\n", + "4 1.0 103.0 0.004513" ] }, "execution_count": 8, @@ -532,7 +532,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 2.9599234249999427 seconds for prediction.\n" + "Took 3.0358772649997263 seconds for prediction.\n" ] } ], @@ -564,10 +564,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.046569\n", - "NDCG:\t0.192938\n", - "Precision@K:\t0.173171\n", - "Recall@K:\t0.096173\n" + "MAP:\t0.046273\n", + "NDCG:\t0.190750\n", + "Precision@K:\t0.173277\n", + "Recall@K:\t0.096688\n" ] } ], @@ -610,8 +610,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "HR:\t0.485765\n", - "NDCG:\t0.379814\n" + "HR:\t0.488564\n", + "NDCG:\t0.383339\n" ] } ], @@ -692,7 +692,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 500.5236881850001 seconds for training.\n" + "Took 513.9623115730001 seconds for training.\n" ] } ], @@ -735,7 +735,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 552.2027924220001 seconds for training.\n" + "Took 566.8783325639997 seconds for training.\n" ] } ], @@ -796,7 +796,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 644.399864857 seconds for training.\n" + "Took 655.1110815689999 seconds for training.\n" ] } ], @@ -825,7 +825,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Took 3.0592451869997603 seconds for prediction.\n" + "Took 3.083744528999887 seconds for prediction.\n" ] } ], @@ -857,10 +857,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "MAP:\t0.046286\n", - "NDCG:\t0.186660\n", - "Precision@K:\t0.169035\n", - "Recall@K:\t0.100020\n" + "MAP:\t0.043232\n", + "NDCG:\t0.181301\n", + "Precision@K:\t0.165111\n", + "Recall@K:\t0.094437\n" ] } ], @@ -878,146 +878,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " \n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "map": 0.046568915592114325 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:3: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " This is separate from the ipykernel package so we can avoid doing imports until\n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "ndcg": 0.37981413819338766 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " after removing the cwd from sys.path.\n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "precision": 0.1731707317073171 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " \"\"\"\n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "recall": 0.096173321399973 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " \n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "map2": 0.046286433403948564 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " import sys\n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "ndcg2": 0.1866595787134735 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:8: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " \n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "precision2": 0.1690349946977731 - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/anaconda/envs/recommenders_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:9: DeprecationWarning: Function record is deprecated and will be removed in verison 1.0.0 (current version 0.19.1). Please see `scrapbook.glue` (nteract-scrapbook) as a replacement for this functionality.\n", - " if __name__ == '__main__':\n" - ] - }, - { - "data": { - "application/papermill.record+json": { - "recall2": 0.10002001904879806 - } - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Record results with papermill for tests\n", "pm.record(\"map\", eval_map)\n", From 3f867429fabe359e6169712b9b671a65dd84c52d Mon Sep 17 00:00:00 2001 From: seanytak Date: Wed, 29 Apr 2020 09:31:39 +0000 Subject: [PATCH 14/17] fix output on nni svd surprise --- .../nni_surprise_svd.ipynb | 287 +++++++++++++----- reco_utils/tuning/nni/nni_utils.py | 2 +- 2 files changed, 204 insertions(+), 85 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb index 6b819468a7..e894d204f9 100644 --- a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -42,9 +42,7 @@ "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", "Surprise version: 1.1.0\n", - "NNI version: 1.5\n", - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" + "NNI version: 1.5\n" ] } ], @@ -92,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 2, "metadata": { "tags": [ "parameters" @@ -106,22 +104,22 @@ "SURPRISE_READER = 'ml-100k'\n", "TMP_DIR = tmp_dir.name\n", "NUM_EPOCHS = 30\n", - "MAX_TRIAL_NUM = 100\n", + "MAX_TRIAL_NUM = 10\n", "# time (in seconds) to wait for each tuning experiment to complete\n", "WAITING_TIME = 20\n", - "MAX_RETRIES = 400 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" + "MAX_RETRIES = 40 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" ] }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 11.0kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.3kKB/s]\n" ] }, { @@ -194,7 +192,7 @@ "4 166 346 1.0" ] }, - "execution_count": 61, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -210,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -219,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 5, "metadata": { "scrolled": true }, @@ -266,7 +264,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -307,7 +305,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -331,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -348,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -403,7 +401,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -413,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -427,7 +425,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -450,53 +448,85 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 13, "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "malformed node or string: {'rmse': 1.160248985195997, 'default': 0, 'ndcg_at_k': 0}", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mcheck_metrics_written\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWAITING_TIME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_retries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mMAX_RETRIES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_metrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_trial_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'maximize'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36mget_trials\u001b[0;34m(optimize_mode)\u001b[0m\n\u001b[1;32m 111\u001b[0m trials = [\n\u001b[1;32m 112\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"finalMetricData\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"logPath\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_trials\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 114\u001b[0m ]\n\u001b[1;32m 115\u001b[0m sorted_trials = sorted(\n", - "\u001b[0;32m~/projects/recommenders/reco_utils/tuning/nni/nni_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 111\u001b[0m trials = [\n\u001b[1;32m 112\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"finalMetricData\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"logPath\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mtrial\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_trials\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 114\u001b[0m ]\n\u001b[1;32m 115\u001b[0m sorted_trials = sorted(\n", - "\u001b[0;32m/anaconda/envs/recommenders_gpu/lib/python3.6/ast.py\u001b[0m in \u001b[0;36mliteral_eval\u001b[0;34m(node_or_string)\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'malformed node or string: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/anaconda/envs/recommenders_gpu/lib/python3.6/ast.py\u001b[0m in \u001b[0;36m_convert\u001b[0;34m(node)\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'malformed node or string: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 85\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: malformed node or string: {'rmse': 1.160248985195997, 'default': 0, 'ndcg_at_k': 0}" - ] - } - ], + "outputs": [], "source": [ - "check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", + "# check_metrics_written(wait=WAITING_TIME, max_retries=MAX_RETRIES)\n", "trials, best_metrics, best_params, best_trial_path = get_trials('maximize')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 0.9918669108638116,\n", + " 'ndcg_at_k': 0.05341690918927691,\n", + " 'precision_at_k': 0.051437699680511186}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_metrics" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'parameter_id': 1,\n", + " 'parameter_source': 'algorithm',\n", + " 'parameters': {'n_factors': 50,\n", + " 'init_mean': -0.32199095465587735,\n", + " 'init_std_dev': 0.11455022391401633,\n", + " 'lr_bu': 0.07562149308631293,\n", + " 'lr_bi': 0.0002925566403137119,\n", + " 'lr_pu': 0.04052551251769183,\n", + " 'lr_qi': 0.005161900020236067,\n", + " 'reg_bu': 0.8623789452897435,\n", + " 'reg_bi': 0.5692944661931444,\n", + " 'reg_pu': 0.2579260065274244,\n", + " 'reg_qi': 0.5207340528183084},\n", + " 'parameter_index': 0}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_params" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'/tmp/tmp21mjabn1/experiments/nR67pfst/trials/esAqs'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_trial_path" ] @@ -510,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -526,7 +556,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -544,9 +574,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'rmse': 0.9897488773596576, 'precision_at_k': 0.0769722814498934, 'ndcg_at_k': 0.0850595462545088}\n" + ] + } + ], "source": [ "test_results_tpe = compute_test_results(svd)\n", "print(test_results_tpe)" @@ -579,7 +617,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -594,17 +632,9 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 21, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CompletedProcess(args=['/anaconda/envs/recommenders_gpu/bin/nnictl', 'create', '--config', '/tmp/tmpxs86uz3d/config_svd.yml'], returncode=0)\n" - ] - } - ], + "outputs": [], "source": [ "stop_nni()\n", "with Timer() as time_random:\n", @@ -613,7 +643,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -624,16 +654,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "# Annealing\n", "config['tuner']['builtinTunerName'] = 'Anneal'\n", "if 'classArgs' not in config['tuner']:\n", - " config['tuner']['classArgs'] = {'optimize_mode': 'maximum'}\n", + " config['tuner']['classArgs'] = {'optimize_mode': 'maximize'}\n", "else:\n", - " config['tuner']['classArgs']['optimize_mode'] = 'maximum'\n", + " config['tuner']['classArgs']['optimize_mode'] = 'maximize'\n", " \n", "with open(config_path, 'w') as fp:\n", " fp.write(yaml.dump(config, default_flow_style=False))" @@ -641,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -652,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -663,7 +693,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -675,7 +705,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -686,7 +716,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -705,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -717,7 +747,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -733,7 +763,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -746,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -757,7 +787,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -769,7 +799,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -780,7 +810,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -798,7 +828,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -818,7 +848,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -829,7 +859,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -840,7 +870,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -855,7 +885,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -879,11 +909,100 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rmseprecision_at_kndcg_at_ktime
TPE0.9900.0770.08580.076
Metis0.9440.0760.085105.617
Random Search0.9740.0520.05184.977
Hyperband1.8630.0330.032104.792
Evolution0.9320.0260.025104.768
Annealing0.9610.0150.014105.013
\n", + "
" + ], + "text/plain": [ + " rmse precision_at_k ndcg_at_k time\n", + "TPE 0.990 0.077 0.085 80.076\n", + "Metis 0.944 0.076 0.085 105.617\n", + "Random Search 0.974 0.052 0.051 84.977\n", + "Hyperband 1.863 0.033 0.032 104.792\n", + "Evolution 0.932 0.026 0.025 104.768\n", + "Annealing 0.961 0.015 0.014 105.013" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "res_df.sort_values(by=\"precision_at_k\", ascending=False).round(3)" ] @@ -899,7 +1018,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -909,7 +1028,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ diff --git a/reco_utils/tuning/nni/nni_utils.py b/reco_utils/tuning/nni/nni_utils.py index d79252ff2d..91a470c042 100644 --- a/reco_utils/tuning/nni/nni_utils.py +++ b/reco_utils/tuning/nni/nni_utils.py @@ -109,7 +109,7 @@ def get_trials(optimize_mode): raise ValueError("optimize_mode should equal either minimize or maximize") all_trials = requests.get(NNI_TRIAL_JOBS_URL).json() trials = [ - (ast.literal_eval(eval(trial["finalMetricData"][0]["data"])), trial["logPath"].split(":")[-1]) + (ast.literal_eval(ast.literal_eval(trial['finalMetricData'][0]['data'])), trial["logPath"].split(":")[-1]) for trial in all_trials ] sorted_trials = sorted( From 1902400ec8df17099fc6e4d4675a94bec837a8f8 Mon Sep 17 00:00:00 2001 From: seanytak Date: Wed, 29 Apr 2020 10:39:31 +0000 Subject: [PATCH 15/17] fix notebook outputs for demonstration purposes --- .../nni_ncf.ipynb | 174 ++++++++++++------ .../nni_surprise_svd.ipynb | 2 +- reco_utils/tuning/nni/ncf_training.py | 1 + 3 files changed, 124 insertions(+), 53 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index 699c63ffde..5c158b918a 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -63,7 +63,7 @@ "text": [ "System version: 3.6.10 |Anaconda, Inc.| (default, Mar 25 2020, 23:51:54) \n", "[GCC 7.3.0]\n", - "Surprise version: 1.1.0\n", + "Tensorflow version: 1.12.0\n", "NNI version: 1.5\n" ] } @@ -98,7 +98,7 @@ "from reco_utils.common.constants import SEED as DEFAULT_SEED\n", "\n", "print(\"System version: {}\".format(sys.version))\n", - "print(\"Surprise version: {}\".format(surprise.__version__))\n", + "print(\"Tensorflow version: {}\".format(pkg_resources.get_distribution(\"tensorflow\").version))\n", "print(\"NNI version: {}\".format(pkg_resources.get_distribution(\"nni\").version))\n", "\n", "tmp_dir = TemporaryDirectory()\n", @@ -131,8 +131,8 @@ "MOVIELENS_DATA_SIZE = '100k'\n", "SURPRISE_READER = 'ml-100k'\n", "TMP_DIR = tmp_dir.name\n", - "NUM_EPOCHS = 30\n", - "MAX_TRIAL_NUM = 30\n", + "NUM_EPOCHS = 10\n", + "MAX_TRIAL_NUM = 16\n", "# time (in seconds) to wait for each tuning experiment to complete\n", "WAITING_TIME = 20\n", "MAX_RETRIES = MAX_TRIAL_NUM*4 # it is recommended to have MAX_RETRIES>=4*MAX_TRIAL_NUM" @@ -147,7 +147,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 4.81k/4.81k [00:00<00:00, 10.8kKB/s]\n" + "100%|██████████| 4.81k/4.81k [00:00<00:00, 8.54kKB/s]\n" ] }, { @@ -360,8 +360,8 @@ "outputs": [], "source": [ "ncf_hyper_params = {\n", - " 'n_factors': {\"_type\": \"choice\", \"_value\": [8, 12, 16, 24, 40]},\n", - " 'learning_rate': {\"_type\": \"uniform\", \"_value\": [1e-6, 1]},\n", + " 'n_factors': {\"_type\": \"choice\", \"_value\": [2, 4, 8, 12]},\n", + " 'learning_rate': {\"_type\": \"uniform\", \"_value\": [1e-3, 1e-2]},\n", "}" ] }, @@ -440,7 +440,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -452,7 +452,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -462,18 +462,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 3.2212178182820117,\n", + " 'ndcg_at_k': 0.1439899349063176,\n", + " 'precision_at_k': 0.11633085896076353}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_metrics_ncf" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'parameter_id': 3,\n", + " 'parameter_source': 'algorithm',\n", + " 'parameters': {'n_factors': 12, 'learning_rate': 0.0023365527461525885},\n", + " 'parameter_index': 0}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "best_params_ncf" ] @@ -489,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -498,7 +525,10 @@ " n_users=data.n_users, \n", " n_items=data.n_items,\n", " model_type=\"NeuMF\",\n", - " n_epochs=NUM_EPOCHS, \n", + " n_factors=4,\n", + " layer_sizes=[16,8,4],\n", + " n_epochs=NUM_EPOCHS,\n", + " learning_rate=1e-3, \n", " verbose=True,\n", " seed=DEFAULT_SEED\n", ")\n", @@ -507,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -546,9 +576,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'rmse': 3.2096711022473214,\n", + " 'precision_at_k': 0.11145281018027572,\n", + " 'ndcg_at_k': 0.13550842348404918}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "test_results = compute_test_results(model, train, validation)\n", "test_results" @@ -565,14 +608,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rmseprecision_at_kndcg_at_kname
03.2096710.1114530.135508ncf_baseline
03.2212180.1163310.143990ncf_tuned
\n", + "
" + ], + "text/plain": [ + " rmse precision_at_k ndcg_at_k name\n", + "0 3.209671 0.111453 0.135508 ncf_baseline\n", + "0 3.221218 0.116331 0.143990 ncf_tuned" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def combine_metrics_dicts(*metrics):\n", " df = pd.DataFrame(metrics[0], index=[0])\n", " for metric in metrics[1:]:\n", - " df = df.append(pd.DataFrame(metric, index=[0]))\n", + " df = df.append(pd.DataFrame(metric, index=[0]), sort=False)\n", " return df\n", "\n", "test_results['name'] = 'ncf_baseline'\n", @@ -584,37 +684,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once we select our model based on the validation set, we can test the model's performance on the test set using the best hyperparameters for the best model (in this case we will simply choose the NCF model, your results may differ depending on your own tests). We will do so by retraining the model on both the train and validation sets to predict on the test set" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_and_validation = train.append(validation).reset_index(drop=True)\n", - "model = NCF(\n", - " n_users=data.n_users, \n", - " n_items=data.n_items,\n", - " model_type=\"NeuMF\",\n", - " n_factors=best_params_ncf[\"parameters\"][\"n_factors\"],\n", - " n_epochs=NUM_EPOCHS,\n", - " learning_rate=best_params_ncf[\"parameters\"][\"learning_rate\"],\n", - " verbose=True,\n", - " seed=DEFAULT_SEED\n", - ")\n", - "model.fit(data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "results = compute_test_results(model, train_and_validation, test)\n", - "results" + "Based on the above metrics, we determine that NNI has identified a set of hyperparameters that does demonstrate an improvement on our metrics of interest. In this example, it turned out that an `n_factors` of 12 contributed to a better performance than an `n_factors` of 4. While the difference in `precision_at_k` and `ndcg_at_k` is small, " ] }, { diff --git a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb index e894d204f9..d1ec52d742 100644 --- a/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_surprise_svd.ipynb @@ -1011,7 +1011,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As we see in the table above, _annealing_ performs best with respect to the primary metric (precision@10) that all the tuners optimized. Also the best NDCG@10 is obtained for annealing and correlates well with precision@10. RMSE on the other hand does not correlate well and is not optimized for annealing, since finding the top k recommendations in the right order is a different task from predicting ratings (high and low) accurately. \n", + "As we see in the table above, _TPE_ performs best with respect to the primary metric (precision@10) that all the tuners optimized. Also the best NDCG@10 is obtained for TPE and correlates well with precision@10. RMSE on the other hand does not correlate well and is not optimized for TPE, since finding the top k recommendations in the right order is a different task from predicting ratings (high and low) accurately. \n", "We have also observed that the above ranking of the tuners is not consistent and may change when trying these experiments multiple times. Since some of these tuners rely heavily on randomized sampling, a larger number of trials is required to get more consistent metrics.\n", "In addition, some of the tuning algorithms themselves come with parameters, which can affect their performance." ] diff --git a/reco_utils/tuning/nni/ncf_training.py b/reco_utils/tuning/nni/ncf_training.py index 45fedbcce0..8bc79e470c 100644 --- a/reco_utils/tuning/nni/ncf_training.py +++ b/reco_utils/tuning/nni/ncf_training.py @@ -12,6 +12,7 @@ sys.path.append("../../../") +import reco_utils.evaluation.python_evaluation as evaluation from reco_utils.recommender.ncf.ncf_singlenode import NCF from reco_utils.recommender.ncf.dataset import Dataset as NCFDataset from reco_utils.dataset import movielens From be04df28e72a5ab8a5588509f9d59ba96504bac0 Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 30 Apr 2020 16:22:50 +0000 Subject: [PATCH 16/17] remove old svd references in nni_ncf --- notebooks/04_model_select_and_optimize/nni_ncf.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb index 5c158b918a..23c422899e 100644 --- a/notebooks/04_model_select_and_optimize/nni_ncf.ipynb +++ b/notebooks/04_model_select_and_optimize/nni_ncf.ipynb @@ -7,7 +7,7 @@ "Copyright (c) Microsoft Corporation. All rights reserved.
\n", "Licensed under the MIT License.\n", "
\n", - "# Model Comparison between SVD and NCF Using the Neural Network Intelligence Toolkit" + "# Model Comparison for NCF Using the Neural Network Intelligence Toolkit" ] }, { @@ -303,7 +303,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `script_params` below are the parameters of the training script that are fixed (unlike `hyper_params` which are tuned). In particular, `VERBOSE, BIASED, RANDOM_STATE, NUM_EPOCHS` are parameters used in the [SVD method](../02_model/surprise_svd_deep_dive.ipynb) and `REMOVE_SEEN` removes the training data from the recommended items. " + "The `script_params` below are the parameters of the training script that are fixed (unlike `hyper_params` which are tuned)." ] }, { @@ -435,7 +435,7 @@ "\n", "The `start_nni` function will run the `nnictl create` command. To find the URL for an active experiment you can run `nnictl webui url` on your terminal.\n", "\n", - "In this notebook the SVD and NCF models are trained sequentially on different NNI experiments. While NNI can run two separate experiments simultaneously by adding the `--port ` flag to `nnictl create`, the total training time will probably be the same as running the experiments sequentially since these are CPU bound processes." + "In this notebook the 16 NCF models are trained concurrently in a single experiment with batches of 8. While NNI can run two separate experiments simultaneously by adding the `--port ` flag to `nnictl create`, the total training time will probably be the same as running the batches sequentially since these are CPU bound processes." ] }, { @@ -684,7 +684,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Based on the above metrics, we determine that NNI has identified a set of hyperparameters that does demonstrate an improvement on our metrics of interest. In this example, it turned out that an `n_factors` of 12 contributed to a better performance than an `n_factors` of 4. While the difference in `precision_at_k` and `ndcg_at_k` is small, " + "Based on the above metrics, we determine that NNI has identified a set of hyperparameters that does demonstrate an improvement on our metrics of interest. In this example, it turned out that an `n_factors` of 12 contributed to a better performance than an `n_factors` of 4. While the difference in `precision_at_k` and `ndcg_at_k` is small, NNI has helped us determine that a slightly larger embedding dimension for NCF may be useful for the movielens dataset." ] }, { @@ -716,7 +716,7 @@ "\n", "In practice, an AutoML framework like NNI is just a tool to help you explore a large space of hyperparameters quickly with a pre-described level of randomization. It is recommended that in addition to using NNI one trains baseline models using typical hyperparamter choices (learning rate of 0.005, 0.001 or regularization rates of 0.05, 0.01, etc.) to draw more meaningful comparisons between model performances. This may help determine if a model is overfitting from the tuner or if there is a statistically significant improvement.\n", "\n", - "Another thing to note is the added computational cost required to train models using an AutoML framework. In this case, it takes about 1 minute to train each of the models on a [Standard_NC6 VM](https://docs.microsoft.com/en-us/azure/virtual-machines/nc-series). With this in mind, while NNI can easily train hundreds of models over all hyperparameters for a model, in practice it may be beneficial to choose a subset of the hyperparameters that are deemed most important and to tune those. Too small of a hyperparameter search space may restrict our exploration, but too large may also lead to random noise in the data being exploited by a specific combination of hyperparameters. \n", + "Another thing to note is the added computational cost required to train models using an AutoML framework. In this case, it takes about 6 minutes to train each of the models on a [Standard_NC6 VM](https://docs.microsoft.com/en-us/azure/virtual-machines/nc-series). With this in mind, while NNI can easily train hundreds of models over all hyperparameters for a model, in practice it may be beneficial to choose a subset of the hyperparameters that are deemed most important and to tune those. Too small of a hyperparameter search space may restrict our exploration, but too large may also lead to random noise in the data being exploited by a specific combination of hyperparameters. \n", "\n", "For examples of scaling larger tuning workloads on clusters of machines, see [the notebooks](./README.md) that employ the [Azure Machine Learning service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters). " ] From 4f8a3f1b9ce378707ed7ec6f2317eacf219fb659 Mon Sep 17 00:00:00 2001 From: seanytak Date: Thu, 30 Apr 2020 16:24:36 +0000 Subject: [PATCH 17/17] remove old svd model saving --- reco_utils/tuning/nni/ncf_training.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/reco_utils/tuning/nni/ncf_training.py b/reco_utils/tuning/nni/ncf_training.py index 8bc79e470c..505d59cf91 100644 --- a/reco_utils/tuning/nni/ncf_training.py +++ b/reco_utils/tuning/nni/ncf_training.py @@ -150,9 +150,6 @@ def main(params): logger.debug("Number of epochs %d", params["n_epochs"]) model = ncf_training(params) - # Save NCF model to the output directory for later use - output_dir = os.environ.get("NNI_OUTPUT_DIR") - # surprise.dump.dump(os.path.join(output_dir, "model.dump"), algo=svd) if __name__ == "__main__": @@ -160,7 +157,7 @@ def main(params): tuner_params = nni.get_next_parameter() logger.debug("Hyperparameters: %s", tuner_params) params = vars(get_params()) - # in the case of Hyperband, use STEPS to allocate the number of epochs SVD will run for + # in the case of Hyperband, use STEPS to allocate the number of epochs NCF will run for if "STEPS" in tuner_params: steps_param = tuner_params["STEPS"] params["n_epochs"] = int(np.rint(steps_param))