home depot product search relevance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "#from my_func import *\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "#from sklearn import pipeline, model_selection\n",
    "from sklearn import pipeline, grid_search\n",
    "#from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.ensemble  import  RandomForestRegressor\n",
    "from sklearn.feature_extraction import *\n",
    "from sklearn.feature_extraction.text import *\n",
    "from sklearn.decomposition import  *\n",
    "from sklearn.pipeline import *\n",
    "from sklearn.grid_search import *\n",
    "from sklearn.metrics import mean_squared_error, make_scorer\n",
    "from sklearn.metrics.pairwise import *\n",
    "#from nltk.metrics import edit_distance\n",
    "from nltk.stem.porter import *\n",
    "stemmer = PorterStemmer()\n",
    "#from nltk.stem.snowball import SnowballStemmer #0.003 improvement but takes twice as long as PorterStemmer\n",
    "#stemmer = SnowballStemmer('english')\n",
    "import re\n",
    "#import enchant\n",
    "import random\n",
    "random.seed(2016)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all = pd.read_csv('df_all_feat.csv', encoding=\"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('input/train.csv', encoding=\"ISO-8859-1\")\n",
    "num_train = df_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (num_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('input/test.csv', encoding=\"ISO-8859-1\")\n",
    "df_pro_desc = pd.read_csv('input/product_descriptions.csv')\n",
    "df_attr = pd.read_csv('input/attributes.csv')\n",
    "#df_brand = pd.read_csv('df_all_brand.csv')\n",
    "df_brand = df_attr[df_attr.name == \"MFG Brand Name\"][[\"product_uid\", \"value\"]].rename(columns={\"value\": \"brand\"})\n",
    "\n",
    "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
    "df_all = pd.merge(df_all, df_pro_desc, how='left', on='product_uid')\n",
    "df_all = pd.merge(df_all, df_brand, how='left', on='product_uid')\n",
    "print(\"--- Files Loaded: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_pro_desc = pd.read_csv('input/product_descriptions.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def str_stem(s): \n",
    "    s = re.sub(r\"([a-z])([A-Z])\", r\"\\1 \\2\", s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_pro_tmp = str_stem(df_pro_desc['product_description'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "strNum = {'zero':0,'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9}\n",
    "def str_stem(s): \n",
    "    if isinstance(s, str):\n",
    "        #s = re.sub(r\"([a-z])([A-Z])\", r\"\\1 \\2\", s)\n",
    "        s = re.sub(r\"(\\w)\\.([A-Z])\", r\"\\1 \\2\", s) #Split words with a.A\n",
    "        s = s.lower()\n",
    "        \n",
    "        s = s.replace(\",\",\" \") #could be number / segment later\n",
    "        s = s.replace(\"$\",\" \")\n",
    "        s = s.replace(\"?\",\" \")\n",
    "        s = s.replace(\"-\",\" \")\n",
    "        s = s.replace(\"//\",\"/\")\n",
    "        s = s.replace(\"..\",\".\")\n",
    "        s = s.replace(\" / \",\" \")\n",
    "        s = s.replace(\" \\\\ \",\" \")\n",
    "        s = s.replace(\"'\",\" \")\n",
    "        s = s.replace(\":\",\" \")\n",
    "        s = s.replace(\".\",\" . \")\n",
    "        s = s.replace('\"', \"\")\n",
    "        s = s.replace(\"(\",\" \")\n",
    "        s = s.replace(\")\",\" \")\n",
    "        s = s.replace(\";\",\" \")\n",
    "        s = s.replace(\"&\",\" \")\n",
    "        s = s.replace(\"!\",\" \")\n",
    "        s = s.replace(\"#\",\" \")\n",
    "        s = re.sub(r\"(^\\.|/)\", r\"\", s)\n",
    "        s = re.sub(r\"(\\.|/)$\", r\"\", s)\n",
    "        s = re.sub(r\"([0-9])([a-z])\", r\"\\1 \\2\", s)\n",
    "        s = re.sub(r\"([a-z])([0-9])\", r\"\\1 \\2\", s)\n",
    "        s = s.replace(\" x \",\" xbi \")\n",
    "        s = re.sub(r\"([a-z])( *)\\.( *)([a-z])\", r\"\\1 \\4\", s)\n",
    "        s = re.sub(r\"([a-z])( *)/( *)([a-z])\", r\"\\1 \\4\", s)\n",
    "        s = s.replace(\"*\",\" xbi \")\n",
    "        s = s.replace(\" by \",\" xbi \")\n",
    "        s = re.sub(r\"([0-9])( *)\\.( *)([0-9])\", r\"\\1.\\4\", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(inches|inch|in|')\\.?\", r\"\\1in. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(foot|feet|ft|'')\\.?\", r\"\\1ft. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(pounds|pound|lbs|lb)\\.?\", r\"\\1lb. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(square|sq) ?\\.?(feet|foot|ft)\\.?\", r\"\\1sq.ft. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(cubic|cu) ?\\.?(feet|foot|ft)\\.?\", r\"\\1cu.ft. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(gallons|gallon|gal)\\.?\", r\"\\1gal. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(ounces|ounce|oz)\\.?\", r\"\\1oz. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(centimeters|cm)\\.?\", r\"\\1cm. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(milimeters|mm)\\.?\", r\"\\1mm. \", s)\n",
    "        s = s.replace(\"°\",\" degrees \")\n",
    "        s = re.sub(r\"([0-9]+)( *)(degrees|degree)\\.?\", r\"\\1deg. \", s)\n",
    "        s = s.replace(\" v \",\" volts \")\n",
    "        s = re.sub(r\"([0-9]+)( *)(volts|volt)\\.?\", r\"\\1volt. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(watts|watt)\\.?\", r\"\\1watt. \", s)\n",
    "        s = re.sub(r\"([0-9]+)( *)(amperes|ampere|amps|amp)\\.?\", r\"\\1amp. \", s)\n",
    "\n",
    "        s = s.replace(\"children\", \"kid \")\n",
    "        s = s.replace(\"child\", \"kid \")\n",
    "        s = s.replace(\"kids\", \"kid \")\n",
    "        s = s.replace(\"refrigerators\", \"fridge \")\n",
    "        s = s.replace(\"refrigerator\", \"fridge \")\n",
    "        s = s.replace(\"freezers\", \"fridge \")\n",
    "        s = s.replace(\"freezer\", \"fridge \")\n",
    "        s = s.replace(\"fragrances\", \"perfume \")\n",
    "        s = s.replace(\"fragrance\", \"perfume \")\n",
    "        s = s.replace(\"perfumes\", \"perfume \")\n",
    "        s = s.replace(\"coffeemaker\", \"coffee maker\")\n",
    "        #s = (\" \").join([z for z in s.split(\" \") if z not in stop_w])\n",
    "        s = (\" \").join([str(strNum[z]) if z in strNum else z for z in s.split(\" \")])\n",
    "        s = (\" \").join([stemmer.stem(z) for z in s.split(\" \")])\n",
    "        \n",
    "        #s = s.lower()\n",
    "        s = s.replace(\"toliet\",\"toilet\")\n",
    "        s = s.replace(\"airconditioner\",\"air conditioner\")\n",
    "        s = s.replace(\"vinal\",\"vinyl\")\n",
    "        s = s.replace(\"vynal\",\"vinyl\")\n",
    "        s = s.replace(\"skill\",\"skil\")\n",
    "        s = s.replace(\"snowbl\",\"snow bl\")\n",
    "        s = s.replace(\"plexigla\",\"plexi gla\")\n",
    "        s = s.replace(\"rustoleum\",\"rust-oleum\")\n",
    "        s = s.replace(\"whirpool\",\"whirlpool\")\n",
    "        s = s.replace(\"whirlpoolga\", \"whirlpool ga\")\n",
    "        s = s.replace(\"whirlpoolstainless\",\"whirlpool stainless\")\n",
    "\n",
    "        s = s.replace(\" . \",\" \")\n",
    "        s = s.replace(\"  \",\" \")\n",
    "        s = s.replace(\"  \",\" \")\n",
    "        return s\n",
    "    else:\n",
    "        return \"null\"\n",
    "\n",
    "def seg_words(str1, str2):\n",
    "    str2 = [z for z in set(str2.split()) if len(z)>2]\n",
    "    words = str1.lower().split(\" \")\n",
    "    s = []\n",
    "    for word in words:\n",
    "        if len(word)>3:\n",
    "            s1 = []\n",
    "            s1 += segmentit(word,str2,True)\n",
    "            if len(s)>1:\n",
    "                s += [z for z in s1 if z not in ['er','ing','s','less'] and len(z)>1]\n",
    "            else:\n",
    "                s.append(word)\n",
    "        else:\n",
    "            s.append(word)\n",
    "    return (\" \".join(s))\n",
    "\n",
    "def segmentit(s, txt_arr, t):\n",
    "    st = s\n",
    "    r = []\n",
    "    for j in range(len(s)):\n",
    "        for word in txt_arr:\n",
    "            if word == s[:-j]:\n",
    "                r.append(s[:-j])\n",
    "                #print(s[:-j],s[len(s)-j:])\n",
    "                s=s[len(s)-j:]\n",
    "                r += segmentit(s, txt_arr, False)\n",
    "    if t:\n",
    "        i = len((\"\").join(r))\n",
    "        if not i==len(st):\n",
    "            r.append(st[i:])\n",
    "    return r\n",
    "\n",
    "def str_common_word(str1, str2):\n",
    "    words, cnt = str1.split(), 0\n",
    "    for word in words:\n",
    "        if str2.find(word)>=0:\n",
    "            cnt+=1\n",
    "    return cnt\n",
    "\n",
    "def str_whole_word(str1, str2, i_):\n",
    "    cnt = 0\n",
    "    while i_ < len(str2):\n",
    "        i_ = str2.find(str1, i_)\n",
    "        if i_ == -1:\n",
    "            return cnt\n",
    "        else:\n",
    "            cnt += 1\n",
    "            i_ += len(str1)\n",
    "    return cnt\n",
    "\n",
    "def str_brand(str1, str2):\n",
    "    if str1 == \"null\":\n",
    "        for word in str2:\n",
    "            for b in brand:\n",
    "                if word in b and b in str2:\n",
    "                    return b\n",
    "    return str1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all['product_description'] = df_all['product_description'].map(lambda x:str_stem(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all = pd.merge(df_all, df_pro_desc, how='left', on='product_uid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all['product_description'] = df_all['product_description'].map(lambda x:repl(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['product_info'] = df_all['search_term']+\"\\t\"+df_all['product_title'] +\"\\t\"+df_all['product_description']\n",
    "df_all['len_of_description'] = df_all['product_description'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "df_all['query_in_description'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\\t')[0],x.split('\\t')[2],0))\n",
    "df_all['query_last_word_in_description'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0].split(\" \")[-1],x.split('\\t')[2]))\n",
    "df_all['word_in_description'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0],x.split('\\t')[2]))\n",
    "df_all['ratio_description'] = df_all['word_in_description']/df_all['len_of_query']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['search_term'] = df_all['search_term'].map(lambda x:str_stem(x))\n",
    "df_all['product_title'] = df_all['product_title'].map(lambda x:str_stem(x))\n",
    "df_all['product_description'] = df_all['product_description'].map(lambda x:str_stem(x))\n",
    "#df_all['brand'] = df_all['brand'].map(lambda x:str_stem(x))\n",
    "df_all['brand'] = df_brand['brand']\n",
    "print(\"--- Stemming: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['product_info'] = df_all['search_term']+\"\\t\"+df_all['product_title'] +\"\\t\"+df_all['product_description']\n",
    "print(\"--- Prod Info: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['len_of_query'] = df_all['search_term'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "df_all['len_of_title'] = df_all['product_title'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "df_all['len_of_description'] = df_all['product_description'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "df_all['len_of_brand'] = df_all['brand'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "print(\"--- Len of: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['search_term'] = df_all['product_info'].map(lambda x:seg_words(x.split('\\t')[0],x.split('\\t')[1]))\n",
    "#print(\"--- Search Term Segment: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['query_in_title'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\\t')[0],x.split('\\t')[1],0))\n",
    "df_all['query_in_description'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\\t')[0],x.split('\\t')[2],0))\n",
    "print(\"--- Query In: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['query_last_word_in_title'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0].split(\" \")[-1],x.split('\\t')[1]))\n",
    "df_all['query_last_word_in_description'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0].split(\" \")[-1],x.split('\\t')[2]))\n",
    "print(\"--- Query Last Word In: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "df_all['word_in_title'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0],x.split('\\t')[1]))\n",
    "df_all['word_in_description'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0],x.split('\\t')[2]))\n",
    "df_all['ratio_title'] = df_all['word_in_title']/df_all['len_of_query']\n",
    "df_all['ratio_description'] = df_all['word_in_description']/df_all['len_of_query']\n",
    "df_all['attr'] = df_all['search_term']+\"\\t\"+df_all['brand']\n",
    "df_all['word_in_brand'] = df_all['attr'].map(lambda x:str_common_word(x.split('\\t')[0],x.split('\\t')[1]))\n",
    "df_all['ratio_brand'] = df_all['word_in_brand']/df_all['len_of_brand']\n",
    "df_brand = pd.unique(df_all.brand.ravel())\n",
    "d={}\n",
    "i = 1000\n",
    "for s in df_brand:\n",
    "    d[s]=i\n",
    "    i+=3\n",
    "df_all['brand_feature'] = df_all['brand'].map(lambda x:d[x])\n",
    "df_all['search_term_feature'] = df_all['search_term'].map(lambda x:len(x))\n",
    "#df_all.to_csv('df_all.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_query= pd.unique(df_all.search_term.ravel())\n",
    "q={}\n",
    "i = 1\n",
    "for s in df_query:\n",
    "    q[s]=i\n",
    "    i+=3\n",
    "df_all['query_feature'] = df_all['search_term'].map(lambda x:q[x])    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for s in df_brand:\n",
    "    d[s]=i\n",
    "    i+=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfvect = TfidfVectorizer(use_idf=True, stop_words = 'english')\n",
    "wholeword = df_all['search_term']+\" \"+df_all['product_title'] +\" \"+df_all['product_description']+ \" \"+df_all['product_attributes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab = tfvect.fit_transform(wholeword)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "st = tfvect.transform(df_all['search_term'])\n",
    "pt = tfvect.transform(df_all['product_title'])\n",
    "pde = tfvect.transform(df_all['product_description'])\n",
    "pa = tfvect.transform(df_all['product_attributes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (st.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['cossim_query_title'] = list(map(lambda x,y: cosine_similarity(x, y), st, pt))\n",
    "cosime_q_t = df_all['cossim_query_title'].astype('float64', coerce=True)\n",
    "df_all['cossim_query_title']=cosime_q_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['cossim_query_description'] = list(map(lambda x,y: cosine_similarity(x, y), st, pde))\n",
    "consime_q_de = df_all['cossim_query_description']\n",
    "df_all['cossim_query_description'] = consime_q_de.astype('float64', coerce = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all['cossim_query_attribute'] = list(map(lambda x,y: cosine_similarity(x, y), st, pa))\n",
    "consime_q_at = df_all['cossim_query_attribute']\n",
    "df_all['cossim_query_attribute'] = consime_q_at.astype('float64', coerce = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all=df_all.drop(['product_info','attr'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def try_divide(x, y, val=0.0):\n",
    "    if y != 0.0:\n",
    "        val = float(x) / y\n",
    "    return val\n",
    "def JaccardCoef(A, B):\n",
    "    A, B = set(A), set(B)\n",
    "    intersect = len(A.intersection(B))\n",
    "    union = len(A.union(B))\n",
    "    coef = try_divide(intersect, union)\n",
    "    return coef\n",
    "\n",
    "def DiceDist(A, B):\n",
    "    A, B = set(A), set(B)\n",
    "    intersect = len(A.intersection(B))\n",
    "    union = len(A) + len(B)\n",
    "    d = try_divide(2*intersect, union)\n",
    "    return d\n",
    "\n",
    "def compute_dist(A, B, dist=\"jaccard_coef\"):\n",
    "    if dist == \"jaccard_coef\":\n",
    "        d = JaccardCoef(A, B)\n",
    "    elif dist == \"dice_dist\":\n",
    "        d = DiceDist(A, B)\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['jaccardcoef_query_title'] = list(df_all.apply(lambda x: compute_dist(x['search_term'], x['product_title'], \"jaccard_coef\"), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all['jaccardcoef_query_attribute'] = list(df_all.apply(lambda x: compute_dist(x['search_term'], x['product_attributes'], \"jaccard_coef\"), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame({ \"dist\": df_all['jaccardcoef_query_attribute']}).to_csv('jaccardcoef_query_attr.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "def cs_retr(x1, x2):\n",
    "    return cosine_similarity(x1, x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'a' :[st], 'b' : [pt], 'c' : [pde], 'd' : [pa]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df['st-pt'] = df['a'].map(lambda x: cs_retr(, x['b']), df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (df_all['cos_sim_st_pt'][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_attr = pd.read_csv(\"input/attributes.csv\", encoding=\"ISO-8859-1\")\n",
    "df_attr.dropna(how=\"all\", inplace=True)\n",
    "df_attr[\"product_uid\"] = df_attr[\"product_uid\"].astype(int)\n",
    "df_attr[\"value\"] = df_attr[\"value\"].astype(str)\n",
    "\n",
    "def concate_attrs(attrs):\n",
    "    names = attrs[\"name\"]\n",
    "    values = attrs[\"value\"]\n",
    "    pairs  = []\n",
    "    for n, v in zip(names, values):\n",
    "        pairs.append(' '.join((n, v)))\n",
    "    return ' '.join(pairs)\n",
    "\n",
    "product_attrs = df_attr.groupby(\"product_uid\").apply(concate_attrs)\n",
    "\n",
    "product_attrs = product_attrs.reset_index(name=\"product_attributes\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cooccurrence_terms(lst1, lst2, join_str):\n",
    "    terms = [\"\"] * len(lst1) * len(lst2)\n",
    "    cnt =  0\n",
    "    for item1 in lst1:\n",
    "        for item2 in lst2:\n",
    "            terms[cnt] = item1 + join_str + item2\n",
    "            cnt += 1\n",
    "    res = \" \".join(terms)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "join_str = \"X\"\n",
    "df_all[\"query_title\"] = list(df_all.apply(lambda x: cooccurrence_terms(x[\"query_unigram\"], x[\"title_unigram\"], join_str), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "token_pattern = r\"(?u)\\b\\w\\w+\\b\"\n",
    "def preprocess_data(line,\n",
    "                    token_pattern=token_pattern,\n",
    "                    encode_digit=False):\n",
    "    token_pattern = re.compile(token_pattern, flags = re.UNICODE | re.LOCALE)\n",
    "    ## tokenize\n",
    "    tokens = [x.lower() for x in token_pattern.findall(line)]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all[\"query_unigram\"] = list(df_all.apply(lambda x: preprocess_data(x[\"search_term\"]), axis=1))\n",
    "#df_all[\"title_unigram\"] = list(df_all.apply(lambda x: preprocess_data(x[\"product_title\"]), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all[\"description_unigram\"] = list(df_all.apply(lambda x: preprocess_data(x[\"product_description\"]), axis=1))\n",
    "df_all[\"attribute_unigram\"] = list(df_all.apply(lambda x: preprocess_data(x[\"product_attributes\"]), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all[\"query_description\"] = list(df_all.apply(lambda x: cooccurrence_terms(x[\"query_unigram\"], x[\"description_unigram\"], join_str), axis=1))\n",
    "df_all[\"query_attribute\"] = list(df_all.apply(lambda x: cooccurrence_terms(x[\"query_unigram\"], x[\"attribute_unigram\"], join_str), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all=df_all.drop(['description_unigram','attribute_unigram'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "q_de = df_all[\"query_description\"] \n",
    "q_attr = df_all[\"query_attribute\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all[\"query_description\"] = q_de\n",
    "df_all[\"query_attribute\"] = q_attr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print (df_all.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (df_all[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all=df_all.drop(['query_unigram','title_unigram'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all = pd.merge(df_all, product_attrs, how=\"left\", on=\"product_uid\")\n",
    "df_all['product_attributes'] = df_all['product_attributes'].fillna('na')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['product_attributes'] = df_all['product_attributes'].map(lambda x:repl(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_all['product_info'] = df_all['search_term']+\"\\t\"+df_all['product_title'] +\"\\t\"+df_all['product_description']+\"\\t\"+df_all['product_attributes']\n",
    "df_all['len_of_attributes'] = df_all['product_attributes'].map(lambda x:len(x.split())).astype(np.int64)\n",
    "print (\"len\")\n",
    "df_all['query_in_attributes'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\\t')[0],x.split('\\t')[3],0))\n",
    "print (\"query in attr\")\n",
    "df_all['word_in_attributes'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0],x.split('\\t')[3]))\n",
    "print (\"word in \")\n",
    "df_all['ratio_attributes'] = df_all['word_in_attributes']/df_all['len_of_query']\n",
    "print (\"ratio in \")\n",
    "df_all['query_last_word_in_attributes'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\\t')[0].split(\" \")[-1],x.split('\\t')[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (df_all['product_description'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all.to_csv('df_all_feat.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "product_counts = pd.DataFrame(pd.Series(df_all.groupby([\"product_uid\"]).size(), name=\"product_count\"))\n",
    "df_all = pd.merge(df_all, product_counts, left_on=\"product_uid\", right_index=True, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#df_all = pd.read_csv('df_all.csv', encoding=\"ISO-8859-1\", index_col=0)\n",
    "df_train = df_all.iloc[:num_train]\n",
    "df_test = df_all.iloc[num_train:]\n",
    "id_test = df_test['id']\n",
    "y_train = df_train['relevance'].values\n",
    "X_train =df_train[:]\n",
    "X_test = df_test[:]\n",
    "print(\"--- Features Set: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (df_train[-5:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nltk.download() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences = []  # Initialize an empty list of sentences\n",
    "\n",
    "for info in df_all['search_term']:\n",
    "    sentences += info_to_sentences(info, tokenizer)\n",
    "    \n",
    "for info in df_all['product_title']:\n",
    "    sentences += info_to_sentences(info, tokenizer)\n",
    "    \n",
    "for info in df_all['product_description']:\n",
    "    sentences += info_to_sentences(info, tokenizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import the built-in logging module and configure it so that Word2Vec \n",
    "# creates nice output messages\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "# Set values for various parameters\n",
    "#num_features = 3000    # Word vector dimensionality                      \n",
    "min_word_count = 5   # Minimum word count                        \n",
    "num_workers = 4       # Number of threads to run in parallel\n",
    "context = 10          # Context window size                                                                                    \n",
    "#downsampling = 1e-3   # Downsample setting for frequent words\n",
    "\n",
    "# Initialize and train the model (this will take some time)\n",
    "from gensim.models import word2vec\n",
    "print (\"Training model...\")\n",
    "model = word2vec.Word2Vec(sentences, workers=num_workers, min_count = min_word_count, window = context)\n",
    "\n",
    "# If you don't plan to train the model any further, calling \n",
    "# init_sims will make the model much more memory-efficient.\n",
    "model.init_sims(replace=True)\n",
    "\n",
    "# It can be helpful to create a meaningful model name and \n",
    "# save the model for later use. You can load it later using Word2Vec.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.syn0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (df_all.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words='english')\n",
    "df_all = pd.read_csv('df_all.csv', encoding=\"ISO-8859-1\")\n",
    "all_word = df_all['search_term'] + \" \" + df_all['product_title'] + \" \" + df_all['product_description']\n",
    "all_tfidf.fit_transform(all_word)\n",
    "        \n",
    "class cust_tfidf_vals(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, key):\n",
    "        self.key = key\n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    def transform(self, data_dict):        \n",
    "        return all_tfidf.transform(data_dict[self.key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "clean_train_query = []\n",
    "for info in X_train[\"search_term\"]:\n",
    "    clean_train_query.append( info_to_wordlist( info, remove_stopwords=False ))\n",
    "\n",
    "trainDataVecs = getAvgFeatureVecs( clean_train_query, model, num_features )\n",
    "\n",
    "print (\"Creating average feature vecs for test reviews\")\n",
    "clean_test_query = []\n",
    "for info in X_test[\"search_term\"]:\n",
    "    clean_test_query.append( info_to_wordlist( info, remove_stopwords=False ))\n",
    "\n",
    "testDataVecs = getAvgFeatureVecs( clean_test_query, model, num_features )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "forest = RandomForestRegressor(n_estimators = 500, n_jobs = -1, random_state = 2016, verbose = 1)\n",
    "forest = forest.fit( trainDataVecs, y_train)\n",
    "\n",
    "# Test & extract results \n",
    "result = forest.predict( testDataVecs )\n",
    "\n",
    "pd.DataFrame({\"id\": id_test, \"relevance\": result}).to_csv('submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from my_func import *\n",
    "print (\"finish import\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from imp import reload\n",
    "import my_func\n",
    "reload(my_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "print(\"--- Training Start ---\")\n",
    "rfr = RandomForestRegressor(n_estimators = 500, n_jobs = -1, random_state = 2016, verbose = 1)\n",
    "tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words='english')\n",
    "tsvd = TruncatedSVD(n_components=10, random_state = 2016)\n",
    "clf = pipeline.Pipeline([\n",
    "        ('union', FeatureUnion(\n",
    "                    transformer_list = [\n",
    "                        ('cst',  cust_regression_vals()),  \n",
    "                        #('feat1', cust_number_col(key='query_in_title')),\n",
    "                        #('feat2', cust_number_col(key='query_in_description')),\n",
    "                        ('txt1', pipeline.Pipeline([('s1', cust_txt_col(key='search_term')), ('tfidf1', tfidf), ('tsvd1', tsvd)])),\n",
    "                        ('txt2', pipeline.Pipeline([('s2', cust_txt_col(key='product_title')), ('tfidf2', tfidf), ('tsvd2', tsvd)])),\n",
    "                        #('txt3', pipeline.Pipeline([('s3', cust_txt_col(key='product_description')), ('tfidf3', tfidf), ('tsvd3', tsvd)])),\n",
    "                        ('txt4', pipeline.Pipeline([('s4', cust_txt_col(key='brand')), ('tfidf4', tfidf), ('tsvd4', tsvd)])),\n",
    "                        ('txt5', pipeline.Pipeline([('s5', cust_txt_col(key='query_title')), ('tfidf5', tfidf), ('tsvd5', tsvd)])),\n",
    "                        ('txt6', pipeline.Pipeline([('s6', cust_txt_col(key='query_description')), ('tfidf6', tfidf), ('tsvd6', tsvd)])),\n",
    "                        #('txt7', pipeline.Pipeline([('s7', cust_txt_col(key='query_attribute')), ('tfidf7', tfidf), ('tsvd7', tsvd)])),\n",
    "                        #('txt5', pipeline.Pipeline([('s5', cust_txt_col(key='product_attributes')), ('tfidf5', tfidf), ('tsvd5', tsvd)])),\n",
    "                        ('txt11', pipeline.Pipeline([('s11', cust_tfidf_vals(key='search_term')), ('tsvd11', tsvd)])),\n",
    "                        ('txt12', pipeline.Pipeline([('s12', cust_tfidf_vals(key='product_title')), ('tsvd12', tsvd)])),\n",
    "                        ('txt13', pipeline.Pipeline([('s13', cust_tfidf_vals(key='product_description')), ('tsvd13', tsvd)]))              \n",
    "                        ],\n",
    "                    transformer_weights = {\n",
    "                        'cst': 1.0,\n",
    "                        #'feat1':0.3,\n",
    "                        #'feat2':0.3,\n",
    "                        'txt1': 0.5,\n",
    "                        'txt2': 0.25,\n",
    "                        #'txt3': 0.0,\n",
    "                        'txt4': 0.5,\n",
    "                        'txt5': 1.0,\n",
    "                        'txt6': 1.0,\n",
    "                        #'txt7': 1.0,\n",
    "                        'txt11': 0.1,\n",
    "                        'txt12': 0.1,\n",
    "                        'txt13': 0.05,\n",
    "                        },\n",
    "                n_jobs = -1\n",
    "                )), \n",
    "        ('rfr', rfr)\n",
    "    ])\n",
    "param_grid = {'rfr__max_features': [10], 'rfr__max_depth': [20]}\n",
    "RMSE  = make_scorer(fmean_squared_error, greater_is_better=False)\n",
    "model = grid_search.GridSearchCV(estimator = clf, param_grid = param_grid, n_jobs = -1, cv = 2, verbose = 20, scoring=RMSE)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameters found by grid search:\")\n",
    "print(model.best_params_)\n",
    "print(\"Best CV score:\")\n",
    "print(model.best_score_)\n",
    "print(model.best_score_ + 0.454686256525)\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "#y_pred[y_pred>3]=3\n",
    "#y_pred[y_pred<1]=1\n",
    "pd.DataFrame({\"id\": id_test, \"relevance\": y_pred}).to_csv('submission.csv',index=False)\n",
    "print(\"--- Training & Testing: %s minutes ---\" % round(((time.time() - start_time)/60),2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import various modules for string cleaning\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "def info_to_wordlist( info, remove_stopwords=False ):\n",
    "    # Function to convert a document to a sequence of words,\n",
    "    # optionally removing stop words.  Returns a list of words.\n",
    "    #\n",
    "    # 1. Remove HTML\n",
    "    #  \n",
    "    # 2. Remove non-letters\n",
    "    info = re.sub(\"[^a-zA-Z0-9]\",\" \", info)\n",
    "    #\n",
    "    # 3. Convert words to lower case and split them\n",
    "    words = info.split()\n",
    "    #\n",
    "    # 4. Optionally remove stop words (false by default)\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        words = [w for w in words if not w in stops]\n",
    "    #\n",
    "    # 5. Return a list of words\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print (model.grid_scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import text\n",
    "import nltk\n",
    "\n",
    "df_outliers = pd.read_csv('df_all.csv', encoding=\"ISO-8859-1\", index_col=0)\n",
    "#stop_ = list(text.ENGLISH_STOP_WORDS)\n",
    "stop_ = []\n",
    "d={}\n",
    "for i in range(len(df_outliers)):\n",
    "    s = str(df_outliers['search_term'][i]).lower()\n",
    "    #s = s.replace(\"\\n\",\" \")\n",
    "    #s = re.sub(\"[^a-z]\",\" \", s)\n",
    "    #s = s.replace(\"  \",\" \")\n",
    "    a = set(s.split(\" \"))\n",
    "    for b_ in a:\n",
    "        if b_ not in stop_ and len(b_)>0:\n",
    "            if b_ not in d:\n",
    "                d[b_] = [1,str_common_word(b_, df_outliers['product_title'][i]),str_common_word(b_, df_outliers['brand'][i]),str_common_word(b_, df_outliers['product_description'][i])]\n",
    "            else:\n",
    "                d[b_][0] += 1\n",
    "                d[b_][1] += str_common_word(b_, df_outliers['product_title'][i])\n",
    "                d[b_][2] += str_common_word(b_, df_outliers['brand'][i])\n",
    "                d[b_][3] += str_common_word(b_, df_outliers['product_description'][i])\n",
    "ds2 = pd.DataFrame.from_dict(d,orient='index')\n",
    "ds2.columns = ['count','in title','in brand','in prod']\n",
    "ds2 = ds2.sort_values(by=['count'], ascending=[False])\n",
    "\n",
    "f = open(\"word_review.csv\", \"w\")\n",
    "f.write(\"word|count|in title|in brand|in description\\n\")\n",
    "for i in range(len(ds2)):\n",
    "    f.write(ds2.index[i] + \"|\" + str(ds2[\"count\"][i]) + \"|\" + str(ds2[\"in title\"][i]) + \"|\" + str(ds2[\"in brand\"][i]) + \"|\" + str(ds2[\"in prod\"][i]) + \"\\n\")\n",
    "f.close()\n",
    "print(\"--- Word List Created: %s minutes ---\" % round(((time.time() - start_time)/60),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gensim"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}