3_Preliminary ML Modeling_Class_Specific_ROC_AUC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import json\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Cleaned Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 6)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean = pd.read_csv('data_clean.csv')\n",
    "data_clean.sample(frac=1)\n",
    "Characters_per_rank = (data_clean.Characters.value_counts()).index \n",
    "data_clean = data_clean[:20000]\n",
    "data_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select and Label Dataset for Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\shamabar\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "#truncate dataset based on number of characters we want to verify\n",
    "threshold = 6\n",
    "data_thres = data_clean[data_clean.Characters.isin(Characters_per_rank[:threshold])]\n",
    "\n",
    "# split the clean text in the rows  into list of words\n",
    "tokenized_data = data_thres[\"Lines\"].apply(lambda text: re.split(' ',text))\n",
    "\n",
    "# Label Character\n",
    "label = preprocessing.LabelEncoder()\n",
    "target = label.fit_transform(data_clean['Characters'].astype(str))\n",
    "data_thres['target'] = label.fit_transform(data_thres['Characters'].astype(str))\n",
    "target_thres = data_thres['target'] \n",
    "my_tags = Characters_per_rank[0:threshold]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_thres.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering using Bag of Words and Word2Vec Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16889, 1749)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a BoW with Count Vectorizer \n",
    "\n",
    "\n",
    "def count_vectorizer(data):\n",
    "    vectorizer = CountVectorizer(ngram_range = (1,2), min_df=10)\n",
    "    # call `fit` to build the vocabulary\n",
    "    vectorizer.fit(data)\n",
    "\n",
    "    # call `transform` to convert text to a bag of words\n",
    "    count_vectorizer_result  = vectorizer.transform(data)\n",
    "\n",
    "    #convert to a numpy array to visualize as dataframe\n",
    "    count_vectorizer_result = count_vectorizer_result.toarray()\n",
    "    count_vectorizer_features = pd.DataFrame(count_vectorizer_result , columns = vectorizer.get_feature_names())\n",
    "    count_vectorizer_features.index = data.index\n",
    "    return count_vectorizer_features\n",
    "\n",
    "count_vectorizer_df = count_vectorizer(data_thres['Lines'])\n",
    "count_vectorizer_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16889, 3887)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a BoW with TF-IDF Scheme \n",
    "\n",
    "def tfidf(data):\n",
    "    tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', ngram_range=(1, 2))\n",
    "\n",
    "    tfidf_result = tfidf.fit_transform(data).toarray()\n",
    "    tfidf_features = pd.DataFrame(tfidf_result, columns = tfidf.get_feature_names())\n",
    "    tfidf_features.index = data.index\n",
    "    return tfidf_features\n",
    "\n",
    "\n",
    "tfidf_df = tfidf(data_thres['Lines'])\n",
    "tfidf_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word2Vec with TFIDF weighting scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a class to impolement word2vec averaged with tf-idf weighting scheme\n",
    "\n",
    "class TfidfEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        self.word2weight = None\n",
    "        self.dim = len(word2vec.wv.syn0[0])\n",
    "\n",
    "    def fit(self, X, y = None):\n",
    "        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n",
    "        tfidf.fit(X)\n",
    "        # if a word was never seen - it must be at least as infrequent\n",
    "        # as any of the known words - so the default idf is the max of \n",
    "        # known idf's\n",
    "        max_idf = max(tfidf.idf_)\n",
    "        self.word2weight = defaultdict(\n",
    "            lambda: max_idf,\n",
    "            [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])\n",
    "\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "                np.mean([self.word2vec.wv[w] * self.word2weight[w]\n",
    "                         for w in words if w in self.word2vec.wv] or\n",
    "                        [np.zeros(self.dim)], axis=0)\n",
    "                for words in X\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# let X be a list of tokenized texts (i.e. list of lists of tokens)\n",
    "model = Word2Vec(tokenized_data,min_count=2,size=100,window=5,iter=100)\n",
    "\n",
    "tfidf_word2vec = TfidfEmbeddingVectorizer(model)\n",
    "tfidf_word2vec.fit(tokenized_data)\n",
    "tfidf_doc_vec = tfidf_word2vec.transform(tokenized_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.similarity('dream','realize')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word2Vec with Simple Averaging Scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a class to impolement word2vec with simple averaging scheme\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        # if a text is empty we should return a vector of zeros\n",
    "        # with the same dimensionality as all the other vectors\n",
    "        self.dim = len(word2vec.wv.syn0[0])\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
    "                    or [np.zeros(self.dim)], axis=0)\n",
    "            for words in X\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "word_model = Word2Vec(tokenized_data,min_count=2,size=100,window=5,iter=100)\n",
    "mean_vec_tr = MeanEmbeddingVectorizer(word_model)\n",
    "doc_vec = mean_vec_tr.transform((tokenized_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Modeling Performance with Different BoW\\ Word Embedding Schemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "Eval_score_embedding = {'Count Vectorizer': [], 'TF-IDF':[], 'Word2Vec':[], 'Word2Vec with TF-IDF Weighting':[] }\n",
    "name_embedded_dataframes = {'Count Vectorizer': count_vectorizer_df,'TF-IDF': tfidf_df, 'Word2Vec': doc_vec, 'Word2Vec with TF-IDF Weighting':tfidf_doc_vec}\n",
    "\n",
    "def word_embedding_performance_eval(clf,dataframe):\n",
    "    Eval_score =  cross_val_score(clf,dataframe, target_thres, cv=5)\n",
    "    return  Eval_score.mean() \n",
    "    \n",
    "model =  LogisticRegression(max_iter = 1e6)\n",
    "for dataframe in name_embedded_dataframes.keys():\n",
    "    Eval_score_embedding [dataframe] = word_embedding_performance_eval(model,name_embedded_dataframes[dataframe])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Embedding_Results = pd.DataFrame.from_dict(Eval_score_embedding,orient='index')\n",
    "Embedding_Results.index.name = 'BoW/Word Embedding Method'\n",
    "Embedding_Results.columns = ['Accuracy'] \n",
    "Embedding_Results.sort_values(by = \"Accuracy\", ascending =False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimize Logistic Regression for Best BOW/ Word Embedding Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model =  LogisticRegression(max_iter = 1e3)\n",
    "params_log = { \"C\": [1e-2,1, 10, 1e2,1e6]}\n",
    "\n",
    "#grid_log = GridSearchCV(model, param_grid=params_log, cv=5)\n",
    "#grid_log.fit(tfidf_df,target_thres)\n",
    "\n",
    "#print(grid_log.best_score_)\n",
    "#print(grid_log.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction Accuracy of Other ML Models with TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes with Count Vectorization and TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Accuracy is low when number of characters considered are higher (data is imbalanced, boot strapping may help)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "model = MultinomialNB()\n",
    "print(cross_val_score(model,count_vectorizer_df, target_thres, cv=5).mean())\n",
    "print(cross_val_score(model,tfidf_df, target_thres, cv=5).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GridSearch with Naive Bayes\n",
    "def gridsearchNB(clf, X, y):\n",
    "\n",
    "    #the grid of parameters to search over\n",
    "    alphas = [0.001,0.01,.1, 1, 5, 10, 50]\n",
    "\n",
    "    #Find the best value for alpha and min_df, and the best classifier\n",
    "    best_alpha = None\n",
    "    maxscore=-np.inf\n",
    "    for alpha in alphas:        \n",
    "        clf = MultinomialNB(alpha=alpha)\n",
    "        cvscore = max(cross_val_score(clf, X,y,cv = 5))\n",
    "        if cvscore > maxscore:\n",
    "                maxscore = cvscore \n",
    "                best_alpha = alpha\n",
    "    return  best_alpha, maxscore\n",
    "\n",
    "print(\"Best Case with Count Vectorizer: \", gridsearchNB(model,count_vectorizer_df, target_thres))\n",
    "print(\"Best Case with TF-IDF: \", gridsearchNB(model,tfidf_df,target_thres))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machine: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_svc =SVC()\n",
    "print('Accuracy of Support Vector Machine without Gridsearch:', cross_val_score(model_svc,tfidf_df, target_thres, cv=5).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Cs = [0.001, 0.01, 0.1, 1, 10]\n",
    "gammas = [0.001, 0.01, 0.1, 1]\n",
    "#kernels =  ['rbf','linear']\n",
    "params_svc = {'C': Cs, 'gamma' : gammas}\n",
    "grid_svc = GridSearchCV(SVC(), param_grid = params_svc, cv=5)\n",
    "grid_svc.fit(tfidf_df,target_thres)\n",
    "print('Optimized Modeling Paramters for Support vector Machine', grid_svm.best_params_)\n",
    "print('Accuracy of Support Vector Machine with Gridsearch:', grid_svm.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overall ROC_AUC for Different Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "def roc_auc_classifier(x,y,model):\n",
    "    \n",
    "    x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.2,random_state =42)\n",
    "    model.fit(x_train,y_train)\n",
    "    y_prob = model.predict_proba(x_test)\n",
    "    \n",
    "    macro_roc_auc_ovo = roc_auc_score(y_test, y_prob, multi_class=\"ovr\",average=\"macro\")\n",
    "    \n",
    "    return  macro_roc_auc_ovo \n",
    "    \n",
    "models = {'Classifier': [SVC( probability = True),MultinomialNB(alpha=1), LogisticRegression(max_iter = 1e6)], 'ROC_AUC_Score':[],'Classifier_Name': ['Support Vector Machines','Naive_Bayes','Logistic_Regression']}\n",
    "\n",
    "\n",
    "for model in models['Classifier']:\n",
    "    roc_auc_score  = roc_auc_classifier(tfidf_df, target_thres,model)\n",
    "    models['ROC_AUC_Score'].append(  roc_auc_score  )\n",
    "print('Classifiers:', models['Classifier_Name'])\n",
    "print('ROC_AUC Scores:', models['ROC_AUC_Score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Class Specific ROC_AUC for Imabalanced Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " %%time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "y= label_binarize(target_thres, classes = list(set(target_thres)))\n",
    "n_classes = y.shape[1]\n",
    "# split training and test sets\n",
    "x_train, x_test, y_train, y_test = train_test_split(tfidf_df, y, test_size=.5,\n",
    "                                                    random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    " %%time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "y= label_binarize(target_thres, classes = list(set(target_thres)))\n",
    "n_classes = y.shape[1]\n",
    "# split training and test sets\n",
    "x_train, x_test, y_train, y_test = train_test_split(tfidf_df, y, test_size=.5,\n",
    "                                                    random_state=0)\n",
    "\n",
    "# Learn to predict each class against the other\n",
    "classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,\n",
    "                                 random_state=0))\n",
    "y_score = classifier.fit(x_train, y_train).decision_function(x_test)\n",
    "\n",
    "# Compute ROC curve and ROC area for each class\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "\n",
    "\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "# First aggregate all false positive rates\n",
    "all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plt.figure(figsize = (16,6))\n",
    "lw=2\n",
    "colors = cycle(['aqua', 'darkorange', 'cornflowerblue','deeppink', 'red', 'green','gold','purple','olivedrab','mediumslateblue','gray','lawngreen'])\n",
    "for i, color in zip(range(n_classes), colors):\n",
    "    plt.plot(fpr[i], tpr[i], color=color, lw=2,\n",
    "                 label='ROC curve of {0} (area = {1:0.2f})'\n",
    "                 ''.format(my_tags[i], roc_auc[i]))\n",
    "        \n",
    "plt.plot([0, 1], [0, 1], 'k--', lw=lw)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate',FontSize = 16)\n",
    "plt.ylabel('True Positive Rate',FontSize = 16)\n",
    "plt.title('Some extension of Receiver operating characteristic to multi-class',FontSize = 16)\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Class Specific ROC_AUC for Oversampled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "Sampling =  RandomOverSampler(random_state=0)\n",
    "\n",
    "y= label_binarize(target_thres, classes = list(set(target_thres)))\n",
    "n_classes = y.shape[1]\n",
    "# split training and test sets\n",
    "x_train, x_test, y_train, y_test = train_test_split(tfidf_df, y, test_size=.5,\n",
    "                                                    random_state=0)\n",
    "x_train, y_train = Sampling.fit_sample(x_train,  y_train)\n",
    "print (x_train.shape)\n",
    "\n",
    "# Learn to predict each class against the other\n",
    "classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,\n",
    "                                 random_state=0))\n",
    "y_score = classifier.fit(x_train, y_train).decision_function(x_test)\n",
    "\n",
    "# Compute ROC curve and ROC area for each class\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "# First aggregate all false positive rates\n",
    "all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (16,6))\n",
    "lw=2\n",
    "colors = cycle(['aqua', 'darkorange', 'cornflowerblue','deeppink', 'red', 'green','gold','purple','olivedrab','mediumslateblue','gray','lawngreen'])\n",
    "for i, color in zip(range(n_classes), colors):\n",
    "    plt.plot(fpr[i], tpr[i], color=color, lw=2,\n",
    "                 label='ROC curve of {0} (area = {1:0.2f})'\n",
    "                 ''.format(my_tags[i], roc_auc[i]))\n",
    "        \n",
    "plt.plot([0, 1], [0, 1], 'k--', lw=lw)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate',FontSize = 16)\n",
    "plt.ylabel('True Positive Rate',FontSize = 16)\n",
    "plt.title('Some extension of Receiver operating characteristic to multi-class',FontSize = 16)\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}