cython_Wsabie_latent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import sys\n",
    "from random import randint\n",
    "from data_load import *\n",
    "from cwsabie_inner import *\n",
    "from transE_label import *\n",
    "from gensim.corpora import Dictionary\n",
    "from collections import Counter\n",
    "from gensim.models import Word2Vec\n",
    "%load_ext cython"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Wsabie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "import numpy as np\n",
    "\n",
    "cimport numpy as np\n",
    "from random import randint\n",
    "import sys\n",
    "import cython\n",
    "cdef extern from \"math.h\":\n",
    "    double sqrt(double m)\n",
    "import math\n",
    "from libc.stdlib cimport malloc, free\n",
    "\n",
    "from libc.math cimport exp\n",
    "from libc.math cimport log\n",
    "from gensim.matutils import argsort\n",
    "\n",
    "from libc.string cimport memset\n",
    "import random\n",
    "# scipy <= 0.15\n",
    "\n",
    "import scipy.linalg.blas as fblas\n",
    "ctypedef np.float32_t REAL_t\n",
    "cdef int ONE = 1\n",
    "\n",
    "\n",
    "REAL = np.float32\n",
    "cdef extern from \"/Users/mayk/working/figer/baseline/PLE/Model/warp/voidptr.h\":\n",
    "    void* PyCObject_AsVoidPtr(object obj)\n",
    "DEF MAX_SENTENCE_LEN = 10000\n",
    "ctypedef void (*scopy_ptr) (const int *N, const float *X, const int *incX, float *Y, const int *incY) nogil\n",
    "ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil\n",
    "ctypedef float (*sdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil\n",
    "ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil\n",
    "ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil\n",
    "ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil\n",
    "\n",
    "\n",
    "cdef scopy_ptr scopy = <scopy_ptr>PyCObject_AsVoidPtr(fblas.scopy._cpointer)  # y = x\n",
    "cdef saxpy_ptr saxpy=<saxpy_ptr>PyCObject_AsVoidPtr(fblas.saxpy._cpointer)  # y += alpha * x\n",
    "cdef sdot_ptr sdot=<sdot_ptr>PyCObject_AsVoidPtr(fblas.sdot._cpointer)  # float = dot(x, y)\n",
    "cdef dsdot_ptr dsdot=<dsdot_ptr>PyCObject_AsVoidPtr(fblas.sdot._cpointer)  # double = dot(x, y)\n",
    "cdef snrm2_ptr snrm2=<snrm2_ptr>PyCObject_AsVoidPtr(fblas.snrm2._cpointer)  # sqrt(x^2)\n",
    "cdef sscal_ptr sscal=<sscal_ptr>PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x\n",
    "DEF EXP_TABLE_SIZE = 10000\n",
    "DEF MAX_EXP = 50\n",
    "\n",
    "cdef REAL_t[EXP_TABLE_SIZE] EXP_TABLE\n",
    "cdef REAL_t[EXP_TABLE_SIZE] LOG_TABLE\n",
    "\n",
    "cdef REAL_t ONEF = <REAL_t>1.0\n",
    "\n",
    "# for when fblas.sdot returns a double\n",
    "cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil:\n",
    "    return <REAL_t>dsdot(N, X, incX, Y, incY)\n",
    "\n",
    "# for when fblas.sdot returns a float\n",
    "cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil:\n",
    "    return <REAL_t>sdot(N, X, incX, Y, incY)\n",
    "\n",
    "# for when no blas available\n",
    "cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil:\n",
    "    # not a true full dot()-implementation: just enough for our cases\n",
    "    cdef int i\n",
    "    cdef REAL_t a\n",
    "    a = <REAL_t>0.0\n",
    "    for i from 0 <= i < 50 by 1:\n",
    "        a += X[i] * Y[i]\n",
    "    return a\n",
    "\n",
    "# for when no blas available\n",
    "cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil:\n",
    "    cdef int i\n",
    "    for i from 0 <= i < N[0] by 1:\n",
    "        Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])]\n",
    "cdef REAL_t cvdot(vec1,vec2,size):\n",
    "    cdef int csize = size\n",
    "    f= dsdot(&csize,<REAL_t *>(np.PyArray_DATA(vec1)),&ONE,<REAL_t *>(np.PyArray_DATA(vec2)),&ONE)\n",
    "    return f\n",
    "def csaxpy(vec1,vec2,alpha,size):\n",
    "    cdef int csize = size\n",
    "    cdef float calpha = alpha\n",
    "    f= our_saxpy_noblas(&csize,&calpha,<REAL_t *>(np.PyArray_DATA(vec1)),&ONE,<REAL_t *>(np.PyArray_DATA(vec2)),&ONE)\n",
    "    return f\n",
    "cdef REAL_t crank(int k):\n",
    "    cdef REAL_t loss = 0.\n",
    "    cdef int i = 1\n",
    "    for i in range(1,k+1):\n",
    "        loss += ONEF/i\n",
    "    return loss\n",
    "cdef REAL_t vsum(REAL_t *vec,int *size):\n",
    "    cdef int i\n",
    "    cdef REAL_t product\n",
    "    product = <REAL_t>0.0\n",
    "    for i from 0 <= i < size[0] by 1:\n",
    "        product += vec[i] **2\n",
    "    return sqrt(product)\n",
    "def cnorm(vec):\n",
    "    cdef int size\n",
    "    size  = len(vec)\n",
    "    return vsum(<REAL_t *>(np.PyArray_DATA(vec)),&size)\n",
    "def init():\n",
    "    for i in range(EXP_TABLE_SIZE):\n",
    "        EXP_TABLE[i] = <REAL_t>exp((i / <REAL_t>EXP_TABLE_SIZE * 2 - 1) * MAX_EXP)\n",
    "        EXP_TABLE[i] = <REAL_t>(EXP_TABLE[i] / (EXP_TABLE[i] + 1))\n",
    "#init()\n",
    "\n",
    "\n",
    "def ctrain(As,B,insts,size,lr,gradient,it,Verbose=False):\n",
    "    cdef float error\n",
    "    next_random = 1\n",
    "    error = 0.\n",
    "   # cdef lam = 0.01\n",
    "    \n",
    "    for i,inst in enumerate(insts):\n",
    "        err,next_random =gradient(As,B,inst,size,next_random,lr=lr)\n",
    "      \n",
    "        error += err\n",
    "        if i % 1000 ==0 and Verbose:\n",
    "            sys.stdout.write(\"\\rIteration %d \" % (it)+ \"trained {0:.0f}%\".format(float(i)*100/len(insts))+\" Loss:{0:.2f}\".format(error))\n",
    "            sys.stdout.flush()\n",
    "    if Verbose:\n",
    "        sys.stdout.write(\"\\n\")\n",
    "    return error\n",
    "\n",
    "cdef void divide(REAL_t *vec, const float *alpha, const int *size):\n",
    "    cdef int i\n",
    "    for i from 0 <= i < size[0] by 1:\n",
    "        vec[i] = vec[i]/alpha[0]\n",
    "def cdivide(vec,alpha):\n",
    "    cdef int size\n",
    "    size  = len(vec)\n",
    "    cdef float r = alpha\n",
    "    divide(<REAL_t *>(np.PyArray_DATA(vec)),&r,&size)\n",
    "\n",
    "    \n",
    "def maximum(vec):\n",
    "    maximum = -float('inf')\n",
    "    idx = -1\n",
    "    for i in range(len(vec)):\n",
    "        if maximum < vec[i]:\n",
    "            maximum = vec[i]\n",
    "            idx = i\n",
    "    return idx,maximum\n",
    "\n",
    "def warp_gradient(As,B,inst,size,next_random,lr=0.01):\n",
    "    #print B\n",
    "    #print B[0]-B[9]\n",
    "    cdef unsigned long long  c_next_random = next_random\n",
    "    dA = np.zeros([As.shape[0],size],dtype=REAL)\n",
    "    dB = np.zeros(B.shape,dtype=REAL)\n",
    "\n",
    "    X = [np.sum(A[inst.features],axis=0) for A in As]\n",
    "    cdef REAL_t error = 0.\n",
    "    cdef REAL_t clr = lr\n",
    "    cdef int N,n_sample \n",
    "    cdef int neg_num = len(inst.negative_labels)\n",
    "    cdef REAL_t norm\n",
    "    cdef int cSize = size\n",
    "    cdef REAL_t floats\n",
    "   # ranks = argsort(scores,reverse=True)\n",
    "    M = len(inst.sparse_labels)\n",
    "    for i,l in enumerate(inst.sparse_labels):\n",
    "        max_idx,s1= maximum([cvdot(x,B[l],cSize) for x in X])\n",
    " \n",
    "        N=1\n",
    "        n_sample  = -1\n",
    "        for k in range(neg_num):\n",
    "            c_next_random = random_int32(&c_next_random)\n",
    "            nl = inst.negative_labels[c_next_random%neg_num]#randint(0,neg_num-1)]\n",
    "            neg_idx,s2 = maximum([cvdot(x,B[nl],cSize) for x in X])\n",
    "            if s1 - s2<1:\n",
    "                n_sample = nl\n",
    "                N = k+1\n",
    "                break\n",
    "        if n_sample!=-1:\n",
    "\n",
    "            L = crank(len(inst.negative_labels)/N)#*(crank(M/(ranks[i]+1)))\n",
    "            negL = -L\n",
    "            error += (1+s2-s1)*L\n",
    "\n",
    "            csaxpy(B[l],dA[max_idx],L,cSize)\n",
    "            csaxpy(-B[n_sample],dA[neg_idx],L,cSize)\n",
    "\n",
    "    for f in inst.features:\n",
    "        for i in range(As.shape[0]):\n",
    "            csaxpy(dA[i],As[i][f],clr,cSize)\n",
    "            norm = cnorm(As[i][f])\n",
    "            if norm >1:\n",
    "                cdivide(As[i][f],norm)\n",
    "\n",
    "    \n",
    "    \n",
    "    return error,c_next_random\n",
    "def save_to_text(matrix,output):\n",
    "    shape = matrix.shape\n",
    "    with open(output,'wb') as out:\n",
    "        out.write(\"%d %d\\n\" % (shape))\n",
    "        for row in matrix:\n",
    "            x = \" \".join(map(lambda x:\"{0:.5}\".format(x),row))\n",
    "            out.write(x+\"\\n\")\n",
    "\n",
    "cdef inline unsigned long long random_int32(unsigned long long *next_random) nogil:\n",
    "    next_random[0] = (next_random[0] * <unsigned long long>25214903917ULL + 11) & 281474976710655ULL\n",
    "    return next_random[0]\n",
    "def crand(sed):\n",
    "    cdef unsigned long long csed = sed\n",
    "    return random_int32(&csed)\n",
    "def save2bin(mat,dct,fn):\n",
    "    n,d  = mat.shape\n",
    "    with open(fn,'w') as out:\n",
    "        out.write(\"%d %d\\n\" % (n,d))\n",
    "        for i in range(n):\n",
    "            text = \" \".join(map(str,mat[i]))\n",
    "            out.write(\"%s %s\\n\" %(dct[i],text))\n",
    "def normalize(mat):\n",
    "    for v in mat:\n",
    "        norm = np.linalg.norm(v)\n",
    "        if norm >=1:\n",
    "            v /= norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "in_dir= \"/Users/mayk/working/figer/baseline/PLE/Intermediate/BBN\"\n",
    "a=MentionData('/Users/mayk/working/figer/baseline/PLE/Intermediate/BBN/train_x_new.txt',\n",
    "              \"/Users/mayk/working/figer/baseline/PLE/Intermediate/BBN/train_y.txt\",\n",
    "             in_dir+\"/feature.txt\",in_dir+\"/type.txt\")\n",
    "label_bin  = Word2Vec.load_word2vec_format('/Users/mayk/working/figer/baseline/PLE/Intermediate/BBN/labels.bin')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 trained 100% Loss:253125.52\n",
      "Iteration 1 trained 100% Loss:178785.86\n",
      "Iteration 2 trained 100% Loss:155965.55\n",
      "Iteration 3 trained 100% Loss:143846.16\n",
      "Iteration 4 trained 100% Loss:135390.00\n",
      "Iteration 5 trained 100% Loss:127930.06\n",
      "Iteration 6 trained 100% Loss:122679.48\n",
      "Iteration 7 trained 100% Loss:117701.02\n",
      "Iteration 8 trained 100% Loss:112958.53\n",
      "Iteration 9 trained 100% Loss:110220.31\n",
      "Iteration 10 trained 100% Loss:106399.84\n",
      "Iteration 11 trained 100% Loss:103590.56\n",
      "Iteration 12 trained 100% Loss:100810.35\n",
      "Iteration 13 trained 100% Loss:98670.17\n",
      "Iteration 14 trained 100% Loss:96363.98\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(12)\n",
    "size= 300\n",
    "K=2\n",
    "A= np.random.uniform(-6/np.sqrt(size),6/np.sqrt(size), [K,len(a.feature2id),size]).astype(np.float32)#np.random.rand(len(a.feature2id),size).astype(np.float32)\n",
    "B = np.asarray([label_bin[a.id2label[i]] for i in range(len(label_bin.vocab))],dtype=np.float32)#np.random.normal(0,1.0, [len(a.label2id),size]).astype(np.float32)\n",
    "next_random = 1\n",
    "normalize(A)\n",
    "normalize(B)\n",
    "\n",
    "for i in range(15):\n",
    "    reg_err =0.\n",
    "    #B-=0.15*(B-C)\n",
    "    ctrain(A,B,a.data,size,0.001,warp_gradient,it=i,Verbose=True)\n",
    "for i in range(K):\n",
    "    save2bin(A[i],a.id2feature,\"/Users/mayk/working/figer/baseline/PLE/Results/BBN/warp_As/warp_A_%d.bin\" % (i))\n",
    "\n",
    "save2bin(B,a.id2label,'/Users/mayk/working/figer/baseline/PLE/Results/BBN/warp_B.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A= np.random.uniform(-6/np.sqrt(size),6/np.sqrt(size), [K,len(a.feature2id),size]).astype(np.float32)#np.random.rand(len(a.feature2id),size).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.05660552, -0.28167629, -0.20095788, ..., -0.04172279,\n",
       "         0.24144581,  0.26826933],\n",
       "       [ 0.01179722, -0.16205899, -0.00426593, ..., -0.10529076,\n",
       "         0.245049  , -0.17339016],\n",
       "       [ 0.23352301, -0.18039671, -0.10599529, ..., -0.15253411,\n",
       "         0.29050085, -0.02163795],\n",
       "       ..., \n",
       "       [ 0.32330236, -0.03868889,  0.18012173, ...,  0.05494594,\n",
       "         0.21774139, -0.29889831],\n",
       "       [ 0.1789182 , -0.23977904, -0.3417342 , ..., -0.21547624,\n",
       "         0.19111684, -0.19578032],\n",
       "       [-0.01055121,  0.18243976,  0.04139563, ..., -0.26376632,\n",
       "        -0.15031138, -0.13791315]], dtype=float32)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.32568955,  0.3092328 , -0.30828238, ...,  0.00127495,\n",
       "        -0.22513478,  0.26290956],\n",
       "       [ 0.19257526,  0.32855147, -0.34415814, ...,  0.00249389,\n",
       "        -0.22354276,  0.22557129],\n",
       "       [ 0.16040505, -0.30075097, -0.33582723, ...,  0.03331999,\n",
       "        -0.02405227, -0.32290831],\n",
       "       ..., \n",
       "       [-0.11637408,  0.30286971,  0.28565001, ...,  0.0366358 ,\n",
       "        -0.00080049,  0.30402589],\n",
       "       [ 0.11059088, -0.23062417,  0.04827109, ...,  0.0069234 ,\n",
       "         0.05998977,  0.2990427 ],\n",
       "       [ 0.29318094,  0.17393352,  0.17989096, ...,  0.19187552,\n",
       "         0.18237527, -0.14068848]], dtype=float32)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}