cleaned_reviews - Copy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Preprocessing on Amazon food reviews\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "The Amazon Fine Food Reviews dataset consists of reviews of fine foods from Amazon.<br>\n",
    "\n",
    "Number of reviews: 568,454<br>\n",
    "Number of users: 256,059<br>\n",
    "Number of products: 74,258<br>\n",
    "Timespan: Oct 1999 - Oct 2012<br>\n",
    "Number of Attributes/Columns in data: 10 \n",
    "\n",
    "Attribute Information:\n",
    "\n",
    "1. Id\n",
    "2. ProductId - unique identifier for the product\n",
    "3. UserId - unqiue identifier for the user\n",
    "4. ProfileName\n",
    "5. HelpfulnessNumerator - number of users who found the review helpful\n",
    "6. HelpfulnessDenominator - number of users who indicated whether they found the review helpful or not\n",
    "7. Score - rating between 1 and 5\n",
    "8. Time - timestamp for the review\n",
    "9. Summary - brief summary of the review\n",
    "10. Text - text of the review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using the SQLite Table to read data.\n",
    "con = sqlite3.connect('database.sqlite') \n",
    "\n",
    "\n",
    "#sorting the data by timestamp and filtering only positive and negative reviews i.e. \n",
    "# not taking into consideration those reviews with Score=3\n",
    "timestamp_data = pd.read_sql_query(\"\"\"\n",
    "SELECT *\n",
    "FROM Reviews WHERE Score != 3\n",
    "ORDER BY Time\n",
    "\"\"\", con) \n",
    "\n",
    "\n",
    "# Give reviews with Score>3 a 1 rating, and reviews with a score<3 a 0 rating.\n",
    "def partition(x):\n",
    "    if x < 3:\n",
    "        return 0\n",
    "    return 1\n",
    "\n",
    "#changing reviews with score less than 3 to be positive and vice-versa\n",
    "actualScore = timestamp_data['Score']\n",
    "positiveNegative = actualScore.map(partition) \n",
    "timestamp_data['Score'] = positiveNegative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>150524</td>\n",
       "      <td>0006641040</td>\n",
       "      <td>ACITT7DI6IDDL</td>\n",
       "      <td>shari zychinski</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>939340800</td>\n",
       "      <td>EVERY book is educational</td>\n",
       "      <td>this witty little book makes my son laugh at l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>150501</td>\n",
       "      <td>0006641040</td>\n",
       "      <td>AJ46FKXOVC7NR</td>\n",
       "      <td>Nicholas A Mesiano</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>940809600</td>\n",
       "      <td>This whole series is great way to spend time w...</td>\n",
       "      <td>I can remember seeing the show when it aired o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>451856</td>\n",
       "      <td>B00004CXX9</td>\n",
       "      <td>AIUWLEQ1ADEG5</td>\n",
       "      <td>Elizabeth Medina</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>944092800</td>\n",
       "      <td>Entertainingl Funny!</td>\n",
       "      <td>Beetlejuice is a well written movie ..... ever...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>230285</td>\n",
       "      <td>B00004RYGX</td>\n",
       "      <td>A344SMIA5JECGM</td>\n",
       "      <td>Vincent P. Ross</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>944438400</td>\n",
       "      <td>A modern day fairy tale</td>\n",
       "      <td>A twist of rumplestiskin captured on film, sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>374359</td>\n",
       "      <td>B00004CI84</td>\n",
       "      <td>A344SMIA5JECGM</td>\n",
       "      <td>Vincent P. Ross</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>944438400</td>\n",
       "      <td>A modern day fairy tale</td>\n",
       "      <td>A twist of rumplestiskin captured on film, sta...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id   ProductId          UserId         ProfileName  \\\n",
       "0  150524  0006641040   ACITT7DI6IDDL     shari zychinski   \n",
       "1  150501  0006641040   AJ46FKXOVC7NR  Nicholas A Mesiano   \n",
       "2  451856  B00004CXX9   AIUWLEQ1ADEG5    Elizabeth Medina   \n",
       "3  230285  B00004RYGX  A344SMIA5JECGM     Vincent P. Ross   \n",
       "4  374359  B00004CI84  A344SMIA5JECGM     Vincent P. Ross   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score       Time  \\\n",
       "0                     0                       0      1  939340800   \n",
       "1                     2                       2      1  940809600   \n",
       "2                     0                       0      1  944092800   \n",
       "3                     1                       2      1  944438400   \n",
       "4                     1                       2      1  944438400   \n",
       "\n",
       "                                             Summary  \\\n",
       "0                          EVERY book is educational   \n",
       "1  This whole series is great way to spend time w...   \n",
       "2                               Entertainingl Funny!   \n",
       "3                            A modern day fairy tale   \n",
       "4                            A modern day fairy tale   \n",
       "\n",
       "                                                Text  \n",
       "0  this witty little book makes my son laugh at l...  \n",
       "1  I can remember seeing the show when it aired o...  \n",
       "2  Beetlejuice is a well written movie ..... ever...  \n",
       "3  A twist of rumplestiskin captured on film, sta...  \n",
       "4  A twist of rumplestiskin captured on film, sta...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating subset dataframe of 10000 points\n",
    "sub_data=timestamp_data[0:250000]\n",
    "sub_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing Data and Storing in CSV File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import time\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading stop words from nltk library\n",
    "stop_words = set(stopwords.words('english'))\n",
    "# BeautifulSoup is used to remove html tags from the text\n",
    "from bs4 import BeautifulSoup \n",
    "import re # For regular expressions\n",
    "\n",
    "\n",
    "def nlp_preprocessing(total_text, index, column):\n",
    "    if type(total_text) is not int:\n",
    "        string = \"\"\n",
    "        # 1. Removing html tags\n",
    "        total_text = BeautifulSoup(total_text).get_text()\n",
    "        # replace every special char with space\n",
    "        total_text = re.sub('[^a-zA-Z0-9\\n]', ' ', total_text)\n",
    "        # replace multiple spaces with single space\n",
    "        total_text = re.sub('\\s+',' ', total_text)\n",
    "        # converting all the chars into lower-case.\n",
    "        total_text = total_text.lower()\n",
    "        \n",
    "        for word in total_text.split():\n",
    "        # if the word is a not a stop word then retain that word from the data\n",
    "            if not word in stop_words:\n",
    "                string += word + \" \"\n",
    "        \n",
    "        sub_data[column][index] = string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time took for preprocessing the text : 19560.216840155324 seconds\n"
     ]
    }
   ],
   "source": [
    "#text processing stage.\n",
    "start_time = time.clock()\n",
    "for index, row in sub_data.iterrows():\n",
    "    nlp_preprocessing(row['Text'], index, 'Text')\n",
    "print('Time took for preprocessing the text :',time.clock() - start_time, \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data.to_csv('cleanedreviews.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>150524</td>\n",
       "      <td>0006641040</td>\n",
       "      <td>ACITT7DI6IDDL</td>\n",
       "      <td>shari zychinski</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>939340800</td>\n",
       "      <td>EVERY book is educational</td>\n",
       "      <td>witty little book makes son laugh loud recite ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>150501</td>\n",
       "      <td>0006641040</td>\n",
       "      <td>AJ46FKXOVC7NR</td>\n",
       "      <td>Nicholas A Mesiano</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>940809600</td>\n",
       "      <td>This whole series is great way to spend time w...</td>\n",
       "      <td>remember seeing show aired television years ag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>451856</td>\n",
       "      <td>B00004CXX9</td>\n",
       "      <td>AIUWLEQ1ADEG5</td>\n",
       "      <td>Elizabeth Medina</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>944092800</td>\n",
       "      <td>Entertainingl Funny!</td>\n",
       "      <td>beetlejuice well written movie everything exce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>230285</td>\n",
       "      <td>B00004RYGX</td>\n",
       "      <td>A344SMIA5JECGM</td>\n",
       "      <td>Vincent P. Ross</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>944438400</td>\n",
       "      <td>A modern day fairy tale</td>\n",
       "      <td>twist rumplestiskin captured film starring mic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>374359</td>\n",
       "      <td>B00004CI84</td>\n",
       "      <td>A344SMIA5JECGM</td>\n",
       "      <td>Vincent P. Ross</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>944438400</td>\n",
       "      <td>A modern day fairy tale</td>\n",
       "      <td>twist rumplestiskin captured film starring mic...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      Id   ProductId          UserId         ProfileName  \\\n",
       "0           0  150524  0006641040   ACITT7DI6IDDL     shari zychinski   \n",
       "1           1  150501  0006641040   AJ46FKXOVC7NR  Nicholas A Mesiano   \n",
       "2           2  451856  B00004CXX9   AIUWLEQ1ADEG5    Elizabeth Medina   \n",
       "3           3  230285  B00004RYGX  A344SMIA5JECGM     Vincent P. Ross   \n",
       "4           4  374359  B00004CI84  A344SMIA5JECGM     Vincent P. Ross   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score       Time  \\\n",
       "0                     0                       0      1  939340800   \n",
       "1                     2                       2      1  940809600   \n",
       "2                     0                       0      1  944092800   \n",
       "3                     1                       2      1  944438400   \n",
       "4                     1                       2      1  944438400   \n",
       "\n",
       "                                             Summary  \\\n",
       "0                          EVERY book is educational   \n",
       "1  This whole series is great way to spend time w...   \n",
       "2                               Entertainingl Funny!   \n",
       "3                            A modern day fairy tale   \n",
       "4                            A modern day fairy tale   \n",
       "\n",
       "                                                Text  \n",
       "0  witty little book makes son laugh loud recite ...  \n",
       "1  remember seeing show aired television years ag...  \n",
       "2  beetlejuice well written movie everything exce...  \n",
       "3  twist rumplestiskin captured film starring mic...  \n",
       "4  twist rumplestiskin captured film starring mic...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('cleanedreviews.csv')\n",
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}