Practical 2 (old).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os, glob\n",
    "from smart_open import smart_open\n",
    "\n",
    "dataDir = 'aclImdb'\n",
    "\n",
    "if not os.path.isdir(dataDir):\n",
    "    raise Exception('Download Data')\n",
    "\n",
    "folders = ['train', 'test', 'unsup']\n",
    "sentiments = {'pos': 1, 'neg': -1, 'unsup': 0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data file exists\n",
      "CPU times: user 2.51 ms, sys: 143 µs, total: 2.66 ms\n",
      "Wall time: 393 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from gensim.utils import tokenize as _tokenize\n",
    "from collections import namedtuple\n",
    "import re\n",
    "\n",
    "SentimentDocument = namedtuple('SentimentDocument', 'words tags movie_id sentiment')\n",
    "\n",
    "# return generator of lines for a file\n",
    "def read(f):\n",
    "    with smart_open(f, 'rb') as f:\n",
    "        for l in f:\n",
    "            yield l.decode('utf-8')\n",
    "    # return smart_open(f, 'rb').read().decode('utf-8')\n",
    "\n",
    "def readURL(f):\n",
    "    with smart_open(f, 'rb') as f:\n",
    "        yeild \n",
    "# Generator of generators\n",
    "def flatMap(ls, func=lambda x: x):\n",
    "    return (func(i) for l in ls for i in l)\n",
    "\n",
    "# generator of lines -> generator of tokens\n",
    "def tokenize(lines):\n",
    "    return (token for line in lines for token in _tokenize(line, lowercase=True, deacc=True))\n",
    "\n",
    "urlmatcher = re.compile(r\"http://www.imdb.com/title/(.*)/usercomments\")\n",
    "\n",
    "def genFiles():\n",
    "    for d in folders:\n",
    "        for s, si in sentiments.items():\n",
    "            if not os.path.isdir(\"{}/{}/{}\".format(dataDir, d, s)): continue\n",
    "            filelist = sorted(glob.glob(\"{}/{}/{}/*.txt\".format(dataDir, d, s)))\n",
    "            with smart_open(\"{}/{}/urls_{}.txt\".format(dataDir, d, s), 'r') as urlfile:\n",
    "                urls = (urlmatcher.match(url).group(1) for url in urlfile)\n",
    "                for url, f in zip(urls, filelist):\n",
    "                    yield si, url, f\n",
    "\n",
    "# s and list of filenames generator\n",
    "# fileLists = ((i, sorted(glob.glob(\"{}/{}/{}/*.txt\".format(dataDir, d, s)))) for s, i in sentiments.items() for d in folders)\n",
    "\n",
    "# s and filename generator\n",
    "# files = ((s,f) for s, fs in fileLists for f in fs)\n",
    "\n",
    "# gen of gen of tokens\n",
    "tokenLists = ((s, url, tokenize(read(f))) for s, url, f in genFiles())\n",
    "\n",
    "if not os.path.isfile('all_data.txt'):\n",
    "    with smart_open('all_data.txt', 'wb') as f:\n",
    "        for s, url, tl in tokenLists:\n",
    "            f.write('{} {} {}\\n'.format(s, url,' '.join(tl)).encode('utf-8'))\n",
    "else:\n",
    "    print('data file exists')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.89 s, sys: 166 ms, total: 2.06 s\n",
      "Wall time: 1.85 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from random import shuffle\n",
    "\n",
    "# docs\n",
    "allDocs = []\n",
    "with smart_open('all_data.txt', 'rb') as f:\n",
    "    for i, line in enumerate(f):\n",
    "        l = line.decode('utf-8').split()\n",
    "        allDocs.append(SentimentDocument(l[2:], [i], l[1], int(l[0])))\n",
    "        \n",
    "shuffle(allDocs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "\n",
    "cpuCount = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Models**\n",
    "Fix vector size to 100, and min_count (cutoff) to 3 (as per the Pang&Lee). \n",
    "\n",
    "For each model, ID string = \"model:window_size:epoch\", and store the model to a file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import Doc2Vec\n",
    "from gensim.test.test_doc2vec import ConcatenatedDoc2Vec\n",
    "from collections import OrderedDict\n",
    "\n",
    "archs = ['dbow', 'dmm', 'dmc', 'dbow+dmm', 'dbow+dmc']\n",
    "ws = [10]\n",
    "vs = 100\n",
    "mc = 3\n",
    "s = 0\n",
    "es = [20, 30]\n",
    "\n",
    "readModels = OrderedDict()\n",
    "\n",
    "def getIDstr(arch, w, e):\n",
    "    return \"{}:{}:{}\".format(arch, w, e)\n",
    "\n",
    "def genIDstrs():\n",
    "    for w in ws:\n",
    "        for e in es:\n",
    "            for arch in archs:\n",
    "                yield getIDstr(arch, w, e)\n",
    "\n",
    "def createModel(idstr):\n",
    "    ids = idstr.split(':')\n",
    "    w = int(ids[1])\n",
    "    e = int(ids[2])\n",
    "    if ids[0] == 'dbow':\n",
    "        return Doc2Vec(dm=0, window=w, vector_size=vs, min_count=mc, sample=s, epochs=e, workers=cpuCount)\n",
    "    elif ids[0] == 'dmm':\n",
    "        return Doc2Vec(dm=1, window=w, vector_size=vs, min_count=mc, sample=s, epochs=e, alpha=0.05, comment='alpha=0.05', workers=cpuCount)\n",
    "    elif ids[0] == 'dmc':\n",
    "        return Doc2Vec(dm=1, dm_concat=1, window=w//2, vector_size=vs, sample=s, min_count=mc, epochs=e, workers=cpuCount)\n",
    "    else:\n",
    "        raise Exception('arch does not exist')\n",
    "\n",
    "def createConcatModel(idstr):\n",
    "    ids = idstr.split(':')\n",
    "    w = int(ids[1])\n",
    "    e = int(ids[2])\n",
    "    return ConcatenatedDoc2Vec([getModel(getIDstr(arch, w, e)) for arch in ids[0].split('+')])\n",
    "\n",
    "def getModel(idstr):\n",
    "    if idstr in readModels:\n",
    "        return readModels[idstr]\n",
    "    f = os.path.abspath('models/{}'.format(idstr))\n",
    "    if os.path.isfile(f):\n",
    "        print('loading model {} from file'.format(idstr))\n",
    "        model = Doc2Vec.load(f)\n",
    "    elif '+' in idstr:\n",
    "        print('creating concat model {}'.format(idstr))\n",
    "        model = createConcatModel(idstr)\n",
    "    else:     \n",
    "        print('creating {}'.format(idstr))\n",
    "        model = createModel(idstr)\n",
    "        model.build_vocab(allDocs)\n",
    "        print('training {}'.format(idstr))\n",
    "        model.train(allDocs, total_examples=len(allDocs), epochs=model.epochs)\n",
    "        print('saving {}'.format(idstr))\n",
    "        model.save(f)\n",
    "    readModels[idstr] = model\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model dbow:10:20 from file\n",
      "loading model dmm:10:20 from file\n",
      "loading model dmc:10:20 from file\n",
      "creating concat model dbow+dmm:10:20\n",
      "creating concat model dbow+dmc:10:20\n",
      "loading model dbow:10:30 from file\n",
      "loading model dmm:10:30 from file\n",
      "loading model dmc:10:30 from file\n",
      "creating concat model dbow+dmm:10:30\n",
      "creating concat model dbow+dmc:10:30\n",
      "CPU times: user 5.48 s, sys: 432 ms, total: 5.91 s\n",
      "Wall time: 5.38 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "models = [getModel(idstr) for idstr in genIDstrs()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    " from itertools import islice\n",
    "\n",
    "# Cross Validation\n",
    "\n",
    "def cv(docs):\n",
    "    folds = tuple(docs[i::10] for i in range(10))\n",
    "    return folds[0], [doc for docs in folds[1:] for doc in docs]\n",
    "\n",
    "posDocs = (doc for doc in allDocs if doc.sentiment == 1)\n",
    "negDocs = (doc for doc in allDocs if doc.sentiment == -1)\n",
    "    \n",
    "docs = list(islice(posDocs, 1000))\n",
    "docs.extend(list(islice(negDocs, 1000)))\n",
    "\n",
    "initialTest, initialTrain = cv(docs)\n",
    "test, train = cv(initialTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train SVMs\n",
      "CPU times: user 18.8 s, sys: 4.88 s, total: 23.7 s\n",
      "Wall time: 21.7 s\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import sparse\n",
    "import sys\n",
    "\n",
    "sys.path.append('/home/jaeyeun/Cambridge/nlp/thundersvm/python')\n",
    "from thundersvmScikit import SVC\n",
    "    \n",
    "def modelToSVM(model, trainSet):\n",
    "    svm = SVC(kernel='linear')\n",
    "    train_y = np.array([doc.sentiment for doc in trainSet])\n",
    "    train_X = np.array([model.infer_vector(doc.words) for doc in trainSet])\n",
    "    svm.fit(train_X, train_y)\n",
    "    return svm\n",
    "\n",
    "print('train SVMs')\n",
    "%time initialSVMs = OrderedDict((name, modelToSVM(getModel(name), initialTrain)) for name in genIDstrs())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(svm, model, testSet):\n",
    "    test_X = np.array([model.infer_vector(doc.words) for doc in testSet])\n",
    "    return svm.predict(test_X)\n",
    "\n",
    "def score(svm, model, testSet):\n",
    "    test_X = np.array([model.infer_vector(doc.words) for doc in testSet])\n",
    "    test_y = np.array([doc.sentiment for doc in testSet])\n",
    "    return svm.score(test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of dbow:10:20\n",
      "0.865\n",
      "Accuracy of dmm:10:20\n",
      "0.785\n",
      "Accuracy of dmc:10:20\n",
      "0.735\n",
      "Accuracy of dbow+dmm:10:20\n",
      "0.875\n",
      "Accuracy of dbow+dmc:10:20\n",
      "0.825\n",
      "Accuracy of dbow:10:30\n",
      "0.875\n",
      "Accuracy of dmm:10:30\n",
      "0.765\n",
      "Accuracy of dmc:10:30\n",
      "0.72\n",
      "Accuracy of dbow+dmm:10:30\n",
      "0.865\n",
      "Accuracy of dbow+dmc:10:30\n",
      "0.845\n"
     ]
    }
   ],
   "source": [
    "for idstr in genIDstrs():\n",
    "    print(\"Accuracy of {}\".format(idstr))\n",
    "    model = getModel(idstr)\n",
    "    svm = initialSVMs[idstr]\n",
    "    mean_acc = score(svm, model, initialTest)\n",
    "    print(mean_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8666666666666667\n"
     ]
    }
   ],
   "source": [
    "# the real one\n",
    "finalModel = getModel('dbow+dmc:10:30')\n",
    "finalSVM = modelToSVM(finalModel, train, True)\n",
    "test_X = np.array([finalModel.infer_vector(doc.words) for doc in test])\n",
    "test_y = np.array([doc.sentiment for doc in test])\n",
    "print(finalSVM.score(test_X, test_y))\n",
    "finalResults = finalSVM.predict(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prev\n",
    "from trainingset import TrainingSet\n",
    "from bag import BagOfFrequency\n",
    "from svm import SVM\n",
    "\n",
    "ts = TrainingSet(BagOfFrequency, {1}, 3)\n",
    "for doc in train:\n",
    "    ts.add(doc.sentiment, doc.words)\n",
    "prevSVM = SVM(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8277777777777777\n"
     ]
    }
   ],
   "source": [
    "test_X = [doc.words for doc in test]\n",
    "\n",
    "print(prevSVM.score(test_X, test_y))\n",
    "prevResults = prevSVM.classify(test_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Permutation Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2699460107978404\n",
      "0.26774645070985803\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import cohen_kappa_score as kappa\n",
    "\n",
    "def yesOrNo(actual, results):\n",
    "    assert len(actual) == len(results)\n",
    "    return [1 if actual[i] == results[i] else 0 for i in range(len(actual))]\n",
    "\n",
    "final = yesOrNo(test_y, finalResults)\n",
    "prev = yesOrNo(test_y, prevResults)\n",
    "\n",
    "def permTest(xs, ys, r):\n",
    "    n = len(xs)\n",
    "    diff = np.abs(np.mean(xs) - np.mean(ys))\n",
    "    k = 1\n",
    "    for i in range(r):\n",
    "        rands = np.random.rand(n)\n",
    "        gx, gy = zip(*tuple(((x, y) if rand >= 0.5 else (y, x)) for rand, x, y in zip(rands, xs, ys)))\n",
    "        k += diff <= np.abs(np.mean(gx) - np.mean(gy))\n",
    "    return k / (r + 1)\n",
    "\n",
    "def kappaPermTest(xs, ys, actual, r):\n",
    "    n = len(xs)\n",
    "    diff = np.abs(kappa(xs, actual) - kappa(ys, actual))\n",
    "    k = 1\n",
    "    for i in range(r):\n",
    "        rands = np.random.rand(n)\n",
    "        gx, gy = zip(*tuple(((x, y) if rand >= 0.5 else (y, x)) for rand, x, y in zip(rands, xs, ys)))\n",
    "        k += diff <= np.abs(kappa(gx, actual) - kappa(gy, actual))\n",
    "    return k / (r + 1)\n",
    "\n",
    "p = permTest(final, prev, 5000)\n",
    "print(p)\n",
    "kp = kappaPermTest(finalResults, prevResults, test_y, 5000)\n",
    "print(kp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nlp]",
   "language": "python",
   "name": "conda-env-nlp-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}