From 82b057d909f7acb330261871e87d303b9a5ca266 Mon Sep 17 00:00:00 2001
From: Jiwei Liu <jiweil@ngvpn01-162-98.dyn.scz.us.nvidia.com>
Date: Tue, 16 Oct 2018 20:04:46 -0400
Subject: [PATCH] fix notebook bugs and add running results

---
 python/notebooks/dbscan_demo.ipynb | 115 +++++++++++++++++------
 python/notebooks/pca_demo.ipynb    | 133 +++++++++++++++++++-------
 python/notebooks/tsvd_demo.ipynb   | 146 +++++++++++++++++++++--------
 3 files changed, 290 insertions(+), 104 deletions(-)

diff --git a/python/notebooks/dbscan_demo.ipynb b/python/notebooks/dbscan_demo.ipynb
index f7bb480b97f..57b13d4d9c5 100644
--- a/python/notebooks/dbscan_demo.ipynb
+++ b/python/notebooks/dbscan_demo.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -52,44 +52,53 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def load_data(nrows, ncols, cached = 'data/mortgage.npy'):\n",
+    "import gzip\n",
+    "def load_data(nrows, ncols, cached = 'data/mortgage.npy.gz'):\n",
     "    if os.path.exists(cached):\n",
-    "        X = np.load(cached)\n",
-    "        return X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
+    "        print('use mortgage data')\n",
+    "        with gzip.open(cached) as f:\n",
+    "            X = np.load(f)\n",
+    "        X = X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
     "    else:\n",
-    "        return np.random.rand(nrows,ncols)"
+    "        print('use random data')\n",
+    "        X = np.random.rand(nrows,ncols)\n",
+    "    df = pd.DataFrame({'fea%d'%i:X[:,i] for i in range(X.shape[1])})\n",
+    "    return df"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def np2pygdf(df):\n",
-    "    # convert numpy array to pygdf dataframe\n",
+    "def pd2pygdf(df):\n",
+    "    # convert pandas dataframe to pygdf dataframe\n",
+    "    if isinstance(df,np.ndarray):\n",
+    "        df = pd.DataFrame({'fea%d'%i:df[:,i] for i in range(df.shape[1])})\n",
     "    pdf = pygdf.DataFrame()\n",
-    "    for c in range(df.shape[1]):\n",
-    "        pdf[c] = df[:,c]\n",
+    "    for c,column in enumerate(df):\n",
+    "        pdf[c] = df[column]\n",
     "    return pdf"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def array_equal(a,b,threshold=1e-4,with_sign=True):\n",
+    "from sklearn.metrics import mean_squared_error\n",
+    "def array_equal(a,b,threshold=5e-3,with_sign=True):\n",
     "    a = to_nparray(a)\n",
     "    b = to_nparray(b)\n",
     "    if with_sign == False:\n",
     "        a,b = np.abs(a),np.abs(b)\n",
-    "    res = np.max(np.abs(a-b))<threshold\n",
+    "    res = mean_squared_error(a,b)<threshold\n",
     "    return res\n",
     "\n",
     "def to_nparray(x):\n",
@@ -111,12 +120,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "data (10000, 128)\n",
+      "CPU times: user 24 ms, sys: 4 ms, total: 28 ms\n",
+      "Wall time: 26.9 ms\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
-    "nrows = 1000\n",
+    "nrows = 10000\n",
     "ncols = 128\n",
     "\n",
     "X = load_data(nrows,ncols)\n",
@@ -125,7 +144,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -135,9 +154,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 27.2 s, sys: 72 ms, total: 27.3 s\n",
+      "Wall time: 27 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
     "clustering_sk = skDBSCAN(eps = eps, min_samples = min_samples)\n",
@@ -146,19 +174,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 112 ms, sys: 436 ms, total: 548 ms\n",
+      "Wall time: 547 ms\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
-    "X = np2pygdf(X)"
+    "X = pd2pygdf(X)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 7.66 s, sys: 108 ms, total: 7.77 s\n",
+      "Wall time: 7.79 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
     "clustering_cuml = cumlDBSCAN(eps = eps, min_samples = min_samples)\n",
@@ -167,12 +213,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "compare dbscan: cuml vs sklearn labels_ equal\n"
+     ]
+    }
+   ],
    "source": [
-    "passed = array_equal(clustering_sk.labels_,clustering_cuml.labels_\n",
-    "            threshold=1e-3,with_sign = True)\n",
+    "passed = array_equal(clustering_sk.labels_,clustering_cuml.labels_)\n",
     "message = 'compare dbscan: cuml vs sklearn labels_ %s'%('equal'if passed else 'NOT equal')\n",
     "print(message)"
    ]
@@ -194,7 +247,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.5.5"
   }
  },
  "nbformat": 4,
diff --git a/python/notebooks/pca_demo.ipynb b/python/notebooks/pca_demo.ipynb
index b170b019bef..f846388df52 100644
--- a/python/notebooks/pca_demo.ipynb
+++ b/python/notebooks/pca_demo.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -52,46 +52,54 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
     "import gzip\n",
     "def load_data(nrows, ncols, cached = 'data/mortgage.npy.gz'):\n",
     "    if os.path.exists(cached):\n",
+    "        print('use mortgage data')\n",
     "        with gzip.open(cached) as f:\n",
     "            X = np.load(f)\n",
-    "        return X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
+    "        X = X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
     "    else:\n",
-    "        return np.random.rand(nrows,ncols)"
+    "        print('use random data')\n",
+    "        X = np.random.rand(nrows,ncols)\n",
+    "    df = pd.DataFrame({'fea%d'%i:X[:,i] for i in range(X.shape[1])})\n",
+    "    return df"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def np2pygdf(df):\n",
-    "    # convert numpy array to pygdf dataframe\n",
+    "def pd2pygdf(df):\n",
+    "    # convert pandas dataframe to pygdf dataframe\n",
+    "    if isinstance(df,np.ndarray):\n",
+    "        df = pd.DataFrame({'fea%d'%i:df[:,i] for i in range(df.shape[1])})\n",
     "    pdf = pygdf.DataFrame()\n",
-    "    for c in range(df.shape[1]):\n",
-    "        pdf[c] = df[:,c]\n",
+    "    for c,column in enumerate(df):\n",
+    "        pdf[c] = df[column]\n",
     "    return pdf"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def array_equal(a,b,threshold=1e-4,with_sign=True):\n",
+    "from sklearn.metrics import mean_squared_error\n",
+    "def array_equal(a,b,threshold=2e-3,with_sign=True):\n",
     "    a = to_nparray(a)\n",
     "    b = to_nparray(b)\n",
     "    if with_sign == False:\n",
     "        a,b = np.abs(a),np.abs(b)\n",
-    "    res = np.max(np.abs(a-b))<threshold\n",
+    "    error = mean_squared_error(a,b)\n",
+    "    res = error<threshold\n",
     "    return res\n",
     "\n",
     "def to_nparray(x):\n",
@@ -101,7 +109,7 @@
     "        return np.array([x])\n",
     "    elif isinstance(x,pygdf.DataFrame) or isinstance(x,pygdf.Series):\n",
     "        return x.to_pandas().values\n",
-    "    return x"
+    "    return x    "
    ]
   },
   {
@@ -113,13 +121,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "use mortgage data\n",
+      "data (1048576, 400)\n",
+      "CPU times: user 16.4 s, sys: 2.32 s, total: 18.7 s\n",
+      "Wall time: 18.7 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
     "nrows = 2**20\n",
-    "ncols = 512\n",
+    "ncols = 400\n",
     "\n",
     "X = load_data(nrows,ncols)\n",
     "print('data',X.shape)"
@@ -127,7 +146,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -139,9 +158,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 6min 7s, sys: 1min 37s, total: 7min 44s\n",
+      "Wall time: 21.3 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
     "pca_sk = skPCA(n_components=n_components,svd_solver=svd_solver, \n",
@@ -151,19 +179,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 8.03 s, sys: 2.4 s, total: 10.4 s\n",
+      "Wall time: 3.3 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
-    "X = np2pygdf(X)"
+    "X = pd2pygdf(X)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 3.37 s, sys: 496 ms, total: 3.87 s\n",
+      "Wall time: 3.89 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
     "pca_cuml = cumlPCA(n_components=n_components,svd_solver=svd_solver, \n",
@@ -173,26 +219,43 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "compare pca: cuml vs sklearn          singular_values_ equal\n",
+      "compare pca: cuml vs sklearn               components_ equal\n",
+      "compare pca: cuml vs sklearn       explained_variance_ equal\n",
+      "compare pca: cuml vs sklearn explained_variance_ratio_ equal\n"
+     ]
+    }
+   ],
    "source": [
     "for attr in ['singular_values_','components_','explained_variance_',\n",
     "             'explained_variance_ratio_']:\n",
-    "    passed = array_equal(getattr(pca_sk,attr),getattr(pca_cuml,attr),\n",
-    "        threshold=0.05,with_sign=True)\n",
-    "    message = 'compare pca: cuml vs sklearn %s %s'%(attr,'equal' if passed else 'NOT equal')\n",
+    "    passed = array_equal(getattr(pca_sk,attr),getattr(pca_cuml,attr))\n",
+    "    message = 'compare pca: cuml vs sklearn {:>25} {}'.format(attr,'equal' if passed else 'NOT equal')\n",
     "    print(message)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "compare pca: cuml vs sklearn transformed results equal\n"
+     ]
+    }
+   ],
    "source": [
-    "passed = array_equal(result_sk,result_cuml\n",
-    "            threshold=0.05,with_sign = False)\n",
+    "passed = array_equal(result_sk,result_cuml)\n",
     "message = 'compare pca: cuml vs sklearn transformed results %s'%('equal'if passed else 'NOT equal')\n",
     "print(message)"
    ]
@@ -214,7 +277,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.5.5"
   }
  },
  "nbformat": 4,
diff --git a/python/notebooks/tsvd_demo.ipynb b/python/notebooks/tsvd_demo.ipynb
index 96b865789d4..7bafcce1f70 100644
--- a/python/notebooks/tsvd_demo.ipynb
+++ b/python/notebooks/tsvd_demo.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -52,46 +52,54 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
     "import gzip\n",
     "def load_data(nrows, ncols, cached = 'data/mortgage.npy.gz'):\n",
     "    if os.path.exists(cached):\n",
+    "        print('use mortgage data')\n",
     "        with gzip.open(cached) as f:\n",
     "            X = np.load(f)\n",
-    "        return X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
+    "        X = X[np.random.randint(0,X.shape[0]-1,nrows),:ncols]\n",
     "    else:\n",
-    "        return np.random.rand(nrows,ncols)"
+    "        print('use random data')\n",
+    "        X = np.random.rand(nrows,ncols)\n",
+    "    df = pd.DataFrame({'fea%d'%i:X[:,i] for i in range(X.shape[1])})\n",
+    "    return df"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def np2pygdf(df):\n",
-    "    # convert numpy array to pygdf dataframe\n",
+    "def pd2pygdf(df):\n",
+    "    # convert pandas dataframe to pygdf dataframe\n",
+    "    if isinstance(df,np.ndarray):\n",
+    "        df = pd.DataFrame({'fea%d'%i:df[:,i] for i in range(df.shape[1])})\n",
     "    pdf = pygdf.DataFrame()\n",
-    "    for c in range(df.shape[1]):\n",
-    "        pdf[c] = df[:,c]\n",
+    "    for c,column in enumerate(df):\n",
+    "        pdf[c] = df[column]\n",
     "    return pdf"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def array_equal(a,b,threshold=1e-4,with_sign=True):\n",
+    "from sklearn.metrics import mean_squared_error\n",
+    "def array_equal(a,b,threshold=5e-3,with_sign=True):\n",
     "    a = to_nparray(a)\n",
     "    b = to_nparray(b)\n",
     "    if with_sign == False:\n",
     "        a,b = np.abs(a),np.abs(b)\n",
-    "    res = np.max(np.abs(a-b))<threshold\n",
+    "    error = mean_squared_error(a,b)\n",
+    "    res = error<threshold\n",
     "    return res\n",
     "\n",
     "def to_nparray(x):\n",
@@ -113,13 +121,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "use mortgage data\n",
+      "data (2097152, 40)\n",
+      "CPU times: user 6.21 s, sys: 1.05 s, total: 7.26 s\n",
+      "Wall time: 7.25 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
-    "nrows = 2**20\n",
-    "ncols = 256\n",
+    "nrows = 2**21\n",
+    "ncols = 40\n",
     "\n",
     "X = load_data(nrows,ncols)\n",
     "print('data',X.shape)"
@@ -127,22 +146,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
     "n_components = 10\n",
-    "random_state = 42\n",
-    "algorithm='arpack'"
+    "random_state = 42"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 35.6 s, sys: 3.52 s, total: 39.1 s\n",
+      "Wall time: 1.39 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
+    "algorithm='arpack'\n",
     "tsvd_sk = skTSVD(n_components=n_components,algorithm=algorithm, \n",
     "            random_state=random_state)\n",
     "result_sk = tsvd_sk.fit_transform(X)"
@@ -150,21 +178,40 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 5 s, sys: 768 ms, total: 5.77 s\n",
+      "Wall time: 729 ms\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
-    "X = np2pygdf(X)"
+    "X = pd2pygdf(X)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1.17 s, sys: 412 ms, total: 1.58 s\n",
+      "Wall time: 1.59 s\n"
+     ]
+    }
+   ],
    "source": [
     "%%time\n",
+    "algorithm='full'\n",
     "tsvd_cuml = cumlTSVD(n_components=n_components,algorithm=algorithm, \n",
     "            random_state=random_state)\n",
     "result_cuml = tsvd_cuml.fit_transform(X)"
@@ -172,29 +219,52 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "compare tsvd: cuml vs sklearn          singular_values_ equal\n",
+      "compare tsvd: cuml vs sklearn               components_ equal\n"
+     ]
+    }
+   ],
    "source": [
-    "for attr in ['singular_values_','components_','explained_variance_',\n",
-    "             'explained_variance_ratio_']:\n",
-    "    passed = array_equal(getattr(tsvd_sk,attr),getattr(tsvd_cuml,attr),\n",
-    "        threshold=0.05,with_sign=True)\n",
-    "    message = 'compare tsvd: cuml vs sklearn %s %s'%(attr,'equal' if passed else 'NOT equal')\n",
+    "for attr in ['singular_values_','components_']:\n",
+    "    passed = array_equal(getattr(tsvd_sk,attr),getattr(tsvd_cuml,attr),threshold=0.1)\n",
+    "    # larger error margin due to different algorithms: arpack vs full\n",
+    "    message = 'compare tsvd: cuml vs sklearn {:>25} {}'.format(attr,'equal' if passed else 'NOT equal')\n",
     "    print(message)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "compare tsvd: cuml vs sklearn transformed results equal\n"
+     ]
+    }
+   ],
    "source": [
-    "passed = array_equal(result_sk,result_cuml\n",
-    "            threshold=0.05,with_sign = False)\n",
+    "passed = array_equal(result_sk,result_cuml,threshold=0.1)\n",
+    "# larger error margin due to different algorithms: arpack vs full\n",
     "message = 'compare tsvd: cuml vs sklearn transformed results %s'%('equal'if passed else 'NOT equal')\n",
     "print(message)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {