SVM_data2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ham = [f for f in listdir(\"data/datasets-master/email/plaintext/corpus1/ham\") \\\n",
    "             if isfile(join(\"data\\datasets-master\\email\\plaintext\\corpus1\\ham\", f))]\n",
    "spam = [f for f in listdir(\"data/datasets-master/email/plaintext/corpus1/spam\") \\\n",
    "             if isfile(join(\"data\\datasets-master\\email\\plaintext\\corpus1\\spam\", f))]\n",
    "for index in range(len(ham)):\n",
    "    ham[index] = \"data/datasets-master/email/plaintext/corpus1/ham/\"+ ham[index]\n",
    "for index in range(len(spam)):\n",
    "    spam[index] = \"data/datasets-master/email/plaintext/corpus1/spam/\"+ spam[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "Training_Ham, Testing_Ham = train_test_split(ham, test_size = 0.1, random_state = 1)\n",
    "Training_Spam, Testing_Spam = train_test_split(spam, test_size = 0.1, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3304 368 1350 150\n"
     ]
    }
   ],
   "source": [
    "print(len(Training_Ham), len(Testing_Ham), len(Training_Spam), len(Testing_Spam))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\User\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\User\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk import sent_tokenize, word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import WordNetLemmatizer\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "from nltk.corpus import stopwords\n",
    "s=set(stopwords.words('english'))\n",
    "# s=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import codecs\n",
    "frequency_list = [1, 2, 3, 5, 10, 20, 30, 50, 100, 200, 300, 500, 1000]\n",
    "# frequency_list = [500]\n",
    "\n",
    "dictionary = dict()\n",
    "Training = Training_Ham + Training_Spam\n",
    "Testing = Testing_Ham + Testing_Spam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frequency is 1\n",
      "dict length: 18010\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "91.33\t\t93.2\t\t95.56\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.28\t\t96.5\t\t95.56\n",
      "358 10 13 137\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 2\n",
      "dict length: 11564\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "90.67\t\t93.15\t\t95.37\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.28\t\t96.24\t\t95.37\n",
      "358 10 14 136\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 3\n",
      "dict length: 8633\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "90.67\t\t93.15\t\t95.37\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.28\t\t96.24\t\t95.37\n",
      "358 10 14 136\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 5\n",
      "dict length: 6130\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "91.33\t\t93.2\t\t95.56\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.28\t\t96.5\t\t95.56\n",
      "358 10 13 137\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 10\n",
      "dict length: 3944\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "91.33\t\t93.84\t\t95.75\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.55\t\t96.51\t\t95.75\n",
      "359 9 13 137\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 20\n",
      "dict length: 2395\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "90.67\t\t94.44\t\t95.75\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.83\t\t96.26\t\t95.75\n",
      "360 8 14 136\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 30\n",
      "dict length: 1751\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "91.33\t\t94.48\t\t95.95\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.83\t\t96.51\t\t95.95\n",
      "360 8 13 137\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 50\n",
      "dict length: 1101\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "91.33\t\t93.2\t\t95.56\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "97.28\t\t96.5\t\t95.56\n",
      "358 10 13 137\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 100\n",
      "dict length: 506\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "92.67\t\t90.85\t\t95.17\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "96.2\t\t96.99\t\t95.17\n",
      "354 14 11 139\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 200\n",
      "dict length: 209\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "92.0\t\t87.9\t\t94.02\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "94.84\t\t96.68\t\t94.02\n",
      "349 19 12 138\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 300\n",
      "dict length: 106\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "93.33\t\t83.83\t\t92.86\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "92.66\t\t97.15\t\t92.86\n",
      "341 27 10 140\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 500\n",
      "dict length: 45\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "93.33\t\t77.35\t\t90.15\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "88.86\t\t97.03\t\t90.15\n",
      "327 41 10 140\n",
      "\n",
      "------------------------------------\n",
      "\n",
      "frequency is 1000\n",
      "dict length: 13\n",
      "Spam\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "48.67\t\t82.02\t\t82.05\n",
      "\n",
      "Ham\n",
      "Recall\t\tPrecision\tAccuracy\n",
      "95.65\t\t82.05\t\t82.05\n",
      "352 16 77 73\n",
      "\n",
      "------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "f = open(\"ret/SVM_data2.txt\", \"a\")\n",
    "for frequency in frequency_list:\n",
    "    print('frequency is ' + str(frequency))\n",
    "    f.write('\\nfrequency is ' + str(frequency) + '\\n')\n",
    "    dictionary = dict()\n",
    "    \n",
    "    for file_names in Training:\n",
    "        file = codecs.open(file_names, 'r', encoding='utf-8', errors='ignore')\n",
    "        content = file.read()\n",
    "        find_sub = re.findall(r\"subject : .*\\n\", content)\n",
    "        if len(find_sub) == 0:\n",
    "            find_sub = re.findall(r\"Subject: .*\\n\", content)\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        else:\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        #word = re.findall(r\"[\\w']+\", body) # 93783 ## 21525\n",
    "        word = re.findall(r\"[a-zA-Z']+\", body) # 123656 ## 21224\n",
    "#         word = body.split(' ')\n",
    "#         lemma = WordNetLemmatizer()\n",
    "#         word = [lemma.lemmatize(w, pos = \"v\") for w in word]\n",
    "#         word = [lemma.lemmatize(w, pos = \"n\") for w in word]\n",
    "        for i in word:\n",
    "            dictionary[i] = dictionary.get(i, 0) + 1\n",
    "#     print(len(dictionary))\n",
    "    dictionary = {i:dictionary[i] for i in dictionary if dictionary[i]>frequency}\n",
    "#     print(len(dictionary))\n",
    "    #print(dictionary)\n",
    "    list_dictionary = list(dictionary)\n",
    "    list_dictionary = list(filter(lambda w: not w in s,list_dictionary))\n",
    "    list_dictionary\n",
    "#     print(len(list_dictionary))\n",
    "    \n",
    "    # Remove common word in dictionary\n",
    "    for s__ in s:\n",
    "        if s__ in dictionary:\n",
    "            del dictionary[s__]\n",
    "\n",
    "    print('dict length: ' + str(len(dictionary)))\n",
    "    f.write('dict length: ' + str(len(dictionary)))\n",
    "    count = 0\n",
    "    for i in dictionary:\n",
    "        dictionary[i] = count\n",
    "        count += 1\n",
    "\n",
    "    label_train = [0] * len(Training_Ham) + [1] * len(Training_Spam)\n",
    "    label_test = [0] * len(Testing_Ham) + [1] * len(Testing_Spam)\n",
    "        \n",
    "# ****************** train *****************\n",
    "    train_ndarray = np.zeros((len(Training), len(dictionary)), dtype = int)\n",
    "\n",
    "    count = 0\n",
    "    for Ham_mail in Training_Ham:\n",
    "        file = codecs.open(Ham_mail, 'r', encoding='utf-8', errors='ignore')\n",
    "        content = file.read()\n",
    "        find_sub = re.findall(r\"subject : .*\\n\", content)\n",
    "        if len(find_sub) == 0:\n",
    "            find_sub = re.findall(r\"Subject: .*\\n\", content)\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        else:\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        word = re.findall(r\"[a-zA-Z']+\", body)\n",
    "#         word = body.split(' ')\n",
    "#         lemma = WordNetLemmatizer()\n",
    "#         word = [lemma.lemmatize(w, pos = \"v\") for w in word]\n",
    "#         word = [lemma.lemmatize(w, pos = \"n\") for w in word]\n",
    "        for w in word:\n",
    "            if w in dictionary:\n",
    "                train_ndarray[count][dictionary[w]] = 1\n",
    "        count += 1\n",
    "        \n",
    "    for Spam_mail in Training_Spam:\n",
    "        file = codecs.open(Spam_mail, 'r', encoding='utf-8', errors='ignore')\n",
    "        content = file.read()\n",
    "        find_sub = re.findall(r\"subject : .*\\n\", content)\n",
    "        if len(find_sub) == 0:\n",
    "            find_sub = re.findall(r\"Subject: .*\\n\", content)\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        else:\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        word = re.findall(r\"[a-zA-Z']+\", body)\n",
    "#         word = body.split(' ')\n",
    "#         lemma = WordNetLemmatizer()\n",
    "#         word = [lemma.lemmatize(w, pos = \"v\") for w in word]\n",
    "#         word = [lemma.lemmatize(w, pos = \"n\") for w in word]\n",
    "        for w in word:\n",
    "            if w in dictionary:\n",
    "                train_ndarray[count][dictionary[w]] = 1\n",
    "        count += 1\n",
    "    train_dataframe = pd.DataFrame(train_ndarray)\n",
    "    \n",
    "# ****************** test ******************\n",
    "    \n",
    "    test_ndarray = np.zeros((len(Testing), len(dictionary)), dtype = int)\n",
    "    count = 0\n",
    "    for Ham_mail in Testing_Ham:\n",
    "        file = codecs.open(Ham_mail, 'r', encoding='utf-8', errors='ignore')\n",
    "        content = file.read()\n",
    "        find_sub = re.findall(r\"subject : .*\\n\", content)\n",
    "        if len(find_sub) == 0:\n",
    "            find_sub = re.findall(r\"Subject: .*\\n\", content)\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        else:\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        word = re.findall(r\"[a-zA-Z']+\", body)\n",
    "#         word = body.split(' ')\n",
    "#         lemma = WordNetLemmatizer()\n",
    "#         word = [lemma.lemmatize(w, pos = \"v\") for w in word]\n",
    "#         word = [lemma.lemmatize(w, pos = \"n\") for w in word]\n",
    "        for w in word:\n",
    "            if w in dictionary:\n",
    "                test_ndarray[count][dictionary[w]] = 1\n",
    "        count += 1\n",
    "          \n",
    "    for Spam_mail in Testing_Spam:\n",
    "        file = codecs.open(Spam_mail, 'r', encoding='utf-8', errors='ignore')\n",
    "        content = file.read()\n",
    "        find_sub = re.findall(r\"subject : .*\\n\", content)\n",
    "        if len(find_sub) == 0:\n",
    "            find_sub = re.findall(r\"Subject: .*\\n\", content)\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        else:\n",
    "            body = content.split(find_sub[-1])[1].lower()\n",
    "        word = re.findall(r\"[a-zA-Z']+\", body)\n",
    "#         word = body.split(' ')\n",
    "#         lemma = WordNetLemmatizer()\n",
    "#         word = [lemma.lemmatize(w, pos = \"v\") for w in word]\n",
    "#         word = [lemma.lemmatize(w, pos = \"n\") for w in word]\n",
    "        for w in word:\n",
    "            if w in dictionary:\n",
    "                test_ndarray[count][dictionary[w]] = 1\n",
    "        count += 1\n",
    "    test_dataframe = pd.DataFrame(test_ndarray)\n",
    "    \n",
    "# **************************************************************\n",
    "\n",
    "    SVM = SVC(kernel='linear', probability=True)\n",
    "    SVM.fit(train_dataframe, label_train)\n",
    "\n",
    "    result = SVM.predict(test_dataframe)\n",
    "\n",
    "    \n",
    "    #-------------- from block -----------------\n",
    "    \n",
    "#     Spam_from = dict()\n",
    "#     count = 0\n",
    "#     for Spam_mail in Training_Spam:\n",
    "#         file = codecs.open(Spam_mail, 'r', encoding='utf-8', errors='ignore')\n",
    "#         word = re.findall(r\"From.*\\n\", file.read())[0]\n",
    "#         word_2 = re.findall(r\"[\\w*\\.?]*@[\\w*\\.?]*\", word)\n",
    "#         if not word_2 :\n",
    "#             continue\n",
    "#         Spam_from[word_2[0]] = Spam_from.get(i, 0)    \n",
    "    \n",
    "#     index = 0\n",
    "#     cheat_result = result.copy()\n",
    "#     for mail in Testing:\n",
    "#         file = codecs.open(mail, 'r', encoding='utf-8', errors='ignore')\n",
    "#         word = re.findall(r\"From.*\\n\", file.read())[0]\n",
    "#         word_2 = re.findall(r\"[\\w*\\.?]*@[\\w*\\.?]*\", word)\n",
    "#         if word_2[0] in Spam_from:\n",
    "#             cheat_result[index] = 1\n",
    "#         index += 1\n",
    "    \n",
    "#     eva_fun_2(label_test, cheat_result, 'NN')\n",
    "    \n",
    "    #--------------- end ---------------------\n",
    "    \n",
    "       \n",
    "    tn, fp, fn, tp = confusion_matrix(label_test, result).ravel()\n",
    "\n",
    "    Accuracy = (tp+tn)/(tn+ fp+ fn+ tp)\n",
    "    spam_Precision = (tp)/(fp+ tp)\n",
    "    spam_Recall = (tp)/(fn+ tp)\n",
    "    \n",
    "    ham_Precision = (tn)/(fn + tn)\n",
    "    ham_Recall = (tn)/(fp + tn)\n",
    "    \n",
    "    print('Spam\\nRecall\\t\\tPrecision\\tAccuracy')\n",
    "    f.write('\\n\\nSpam\\nRecall\\t\\tPrecision\\tAccuracy\\n')\n",
    "    print(str(round(spam_Recall*100, 2))+'\\t\\t'+ str(round(spam_Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2)))\n",
    "    f.write(str(round(spam_Recall*100, 2))+'\\t\\t'+ str(round(spam_Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2)))\n",
    "    \n",
    "    print('\\nHam\\nRecall\\t\\tPrecision\\tAccuracy')\n",
    "    f.write('\\n\\nHam\\nRecall\\t\\tPrecision\\tAccuracy\\n')\n",
    "    print(str(round(ham_Recall*100, 2))+'\\t\\t'+ str(round(ham_Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2)))\n",
    "    f.write(str(round(ham_Recall*100, 2))+'\\t\\t'+ str(round(ham_Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2))+ '\\n\\n')\n",
    "    \n",
    "    print(tn, fp, fn, tp)\n",
    "    f.write('tn ' + str(tn) + ' fp ' + str(fp) + ' fn ' + str(fn) + ' tp ' + str(tp))\n",
    "    print('\\n------------------------------------\\n')\n",
    "    f.write('\\n------------------------------------\\n')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352, 16, 77, 73)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# count_list = []\n",
    "# for j in range(len(dictionary)):\n",
    "#     count = 0\n",
    "#     for i in range(len(label_train)):\n",
    "#         if (train_dataframe[j][i] == 1) & (label_train[i] == 1) :\n",
    "#             count += 1\n",
    "#     count_list.append(count)\n",
    "#     if (count > 100):\n",
    "#         print (j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arr = np.ones([(len(label_test))], dtype = bool)\n",
    "# arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# result = SVM.predict(test_dataframe)\n",
    "# l = count_list.copy()\n",
    "# arr = np.ones([(len(label_test))], dtype = bool)\n",
    "# for i in range(3):\n",
    "#     b = np.array(l)\n",
    "#     Max = np.argmax(b)\n",
    "#     l[Max] = -1\n",
    "#     for i in range(len(label_test)):\n",
    "#         if (test_dataframe[Max][i] != 1):\n",
    "#             arr[i] = False\n",
    "# for i in range(len(label_test)):\n",
    "#     if arr[i]:\n",
    "#         result[i] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tn, fp, fn, tp = confusion_matrix(label_test, result).ravel()\n",
    "\n",
    "# Accuracy = (tp+tn)/(tn + fp + fn + tp)\n",
    "# Precision = (tp)/(fp + tp)\n",
    "# Recall = (tp)/(fn + tp)\n",
    "# print('Recall\\t\\tPrecision\\tAccuracy')\n",
    "# # f.write('\\nRecall\\t\\tPrecision\\tAccuracy\\n')\n",
    "# print(str(round(Recall*100, 2))+'\\t\\t'+ str(round(Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2)))\n",
    "# # f.write(str(round(Recall*100, 2))+'\\t\\t'+ str(round(Precision*100, 2))+'\\t\\t'+str(round(Accuracy*100, 2)))\n",
    "# print('\\n------------------------------------\\n')\n",
    "# # f.write('\\n------------------------------------\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.Series(arr).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}