diff --git a/docs/notebooks/doc2vec-IMDB.ipynb b/docs/notebooks/doc2vec-IMDB.ipynb new file mode 100644 index 0000000000..0b0f721be1 --- /dev/null +++ b/docs/notebooks/doc2vec-IMDB.ipynb @@ -0,0 +1,1843 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:26971c428490c5b0358c2d98666355be414831a09bf6cf3c50b03d39bd186505" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "gensim doc2vec & IMDB sentiment dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "TODO: section on introduction & motivation\n", + "\n", + "TODO: prerequisites + dependencies (statsmodels, patsy, ?)" + ] + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Load corpus" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fetch and prep exactly as in Mikolov's go.sh shell script. (Note this cell tests for existence of required files, so steps won't repeat once the final summary file (`aclImdb/alldata-id.txt`) is available alongside this notebook.)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%bash\n", + "# adapted from Mikolov's example go.sh script: \n", + "if [ ! -f \"aclImdb/alldata-id.txt\" ]\n", + "then\n", + " if [ ! -d \"aclImdb\" ] \n", + " then\n", + " if [ ! -f \"aclImdb_v1.tar.gz\" ]\n", + " then\n", + " wget --quiet http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n", + " fi\n", + " tar xf aclImdb_v1.tar.gz\n", + " fi\n", + " \n", + " #this function will convert text to lowercase and will disconnect punctuation and special symbols from words\n", + " function normalize_text {\n", + " awk '{print tolower($0);}' < $1 | sed -e 's/\\./ \\. /g' -e 's/
/ /g' -e 's/\"/ \" /g' \\\n", + " -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\\!/ \\! /g' -e 's/\\?/ \\? /g' \\\n", + " -e 's/\\;/ \\; /g' -e 's/\\:/ \\: /g' > $1-norm\n", + " }\n", + "\n", + " export LC_ALL=C\n", + " for j in train/pos train/neg test/pos test/neg train/unsup; do\n", + " rm temp\n", + " for i in `ls aclImdb/$j`; do cat aclImdb/$j/$i >> temp; awk 'BEGIN{print;}' >> temp; done\n", + " normalize_text temp\n", + " mv temp-norm aclImdb/$j/norm.txt\n", + " done\n", + " mv aclImdb/train/pos/norm.txt aclImdb/train-pos.txt\n", + " mv aclImdb/train/neg/norm.txt aclImdb/train-neg.txt\n", + " mv aclImdb/test/pos/norm.txt aclImdb/test-pos.txt\n", + " mv aclImdb/test/neg/norm.txt aclImdb/test-neg.txt\n", + " mv aclImdb/train/unsup/norm.txt aclImdb/train-unsup.txt\n", + "\n", + " cat aclImdb/train-pos.txt aclImdb/train-neg.txt aclImdb/test-pos.txt aclImdb/test-neg.txt aclImdb/train-unsup.txt > aclImdb/alldata.txt\n", + " awk 'BEGIN{a=0;}{print \"_*\" a \" \" $0; a++;}' < aclImdb/alldata.txt > aclImdb/alldata-id.txt\n", + "fi" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "rm: temp: No such file or directory\n" + ] + } + ], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os.path\n", + "assert os.path.isfile(\"aclImdb/alldata-id.txt\"), \"alldata-id.txt unavailable\"" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The data is small enough to be read into memory. " + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import gensim\n", + "from gensim.models.doc2vec import TaggedDocument\n", + "from collections import namedtuple\n", + "\n", + "SentimentDocument = namedtuple('SentimentDocument', 'words tags split sentiment')\n", + "\n", + "alldocs = [] # will hold all docs in original order\n", + "with open('aclImdb/alldata-id.txt') as alldata:\n", + " for line_no, line in enumerate(alldata):\n", + " tokens = gensim.utils.to_unicode(line).split()\n", + " words = tokens[1:]\n", + " tags = [line_no] # `tags = [tokens[0]]` would also work at extra memory cost\n", + " split = ['train','test','extra','extra'][line_no//25000] # 25k train, 25k test, 25k extra\n", + " sentiment = [1.0, 0.0, 1.0, 0.0, None, None, None, None][line_no//12500] # [12.5K pos, 12.5K neg]*2 then unknown\n", + " alldocs.append(SentimentDocument(words, tags, split, sentiment))\n", + "\n", + "train_docs = [doc for doc in alldocs if doc.split == 'train']\n", + "test_docs = [doc for doc in alldocs if doc.split == 'test']\n", + "doc_list = alldocs[:] # for reshuffling per pass\n", + "\n", + "print('%d docs: %d train-sentiment, %d test-sentiment' % (len(doc_list), len(train_docs), len(test_docs)))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "100000 docs: 25000 train-sentiment, 25000 test-sentiment\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Set-up Doc2Vec Training & Evaluation Models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Approximating experiment of Le & Mikolov [\"Distributed Representations of Sentences and Documents\"](http://cs.stanford.edu/~quocle/paragraph_vector.pdf), also with guidance from Mikolov's [example go.sh](https://groups.google.com/d/msg/word2vec-toolkit/Q49FIrNOQRo/J6KG8mUj45sJ):\n", + "\n", + "`./word2vec -train ../alldata-id.txt -output vectors.txt -cbow 0 -size 100 -window 10 -negative 5 -hs 0 -sample 1e-4 -threads 40 -binary 0 -iter 20 -min-count 1 -sentence-vectors 1`\n", + "\n", + "Parameter choices below vary:\n", + "\n", + "* 100-dimensional vectors, as the 400d vectors of the paper don't seem to offer much benefit on this task\n", + "* similarly, frequent word subsampling seems to decrease sentiment-prediction accuracy, so it's left out\n", + "* `cbow=0` means skip-gram which is equivalent to the paper's 'PV-DBOW' mode, matched in gensim with `dm=0`\n", + "* added to that DBOW model are two DM models, one which averages context vectors (`dm_mean`) and one which concatenates them (`dm_concat`, resulting in a much larger, slower, more data-hungry model)\n", + "* a `min_count=2` saves quite a bit of model memory, discarding only words that appear in a single doc (and are thus no more expressive than the unique-to-each doc vectors themselves)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from gensim.models import Doc2Vec\n", + "import gensim.models.doc2vec\n", + "from collections import OrderedDict\n", + "import multiprocessing\n", + "\n", + "cores = multiprocessing.cpu_count()\n", + "assert gensim.models.doc2vec.FAST_VERSION > -1, \"this will be painfully slow otherwise\"\n", + "\n", + "simple_models = [\n", + " # PV-DM w/concatenation - window=5 (both sides) approximates paper's 10-word total window size\n", + " Doc2Vec(dm=1, dm_concat=1, size=100, window=5, negative=5, hs=0, min_count=2, workers=cores),\n", + " # PV-DBOW \n", + " Doc2Vec(dm=0, size=100, negative=5, hs=0, min_count=2, workers=cores),\n", + " # PV-DM w/average\n", + " Doc2Vec(dm=1, dm_mean=1, size=100, window=10, negative=5, hs=0, min_count=2, workers=cores),\n", + "]\n", + "\n", + "# speed setup by sharing results of 1st model's vocabulary scan\n", + "simple_models[0].build_vocab(alldocs) # PV-DM/concat requires one special NULL word so it serves as template\n", + "print(simple_models[0])\n", + "for model in simple_models[1:]:\n", + " model.reset_from(simple_models[0])\n", + " print(model)\n", + "\n", + "models_by_name = OrderedDict((str(model), model) for model in simple_models)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Doc2Vec(dm/c,d100,n5,w5,mc2,t8)\n", + "Doc2Vec(dbow,d100,n5,mc2,t8)" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "Doc2Vec(dm/m,d100,n5,w10,mc2,t8)" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Following the paper, we also evaluate models in pairs. These wrappers return the concatenation of the vectors from each model. (Only the singular models are trained.)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from gensim.test.test_doc2vec import ConcatenatedDoc2Vec\n", + "models_by_name['dbow+dmm'] = ConcatenatedDoc2Vec([simple_models[1], simple_models[2]])\n", + "models_by_name['dbow+dmc'] = ConcatenatedDoc2Vec([simple_models[1], simple_models[0]])" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Predictive Evaluation Methods" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Helper methods for evaluating error rate." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import numpy as np\n", + "import statsmodels.api as sm\n", + "from random import sample\n", + "\n", + "# for timing\n", + "from contextlib import contextmanager\n", + "from timeit import default_timer\n", + "import time \n", + "\n", + "@contextmanager\n", + "def elapsed_timer():\n", + " start = default_timer()\n", + " elapser = lambda: default_timer() - start\n", + " yield lambda: elapser()\n", + " end = default_timer()\n", + " elapser = lambda: end-start\n", + " \n", + "def logistic_predictor_from_data(train_targets, train_regressors):\n", + " logit = sm.Logit(train_targets, train_regressors)\n", + " predictor = logit.fit(disp=0)\n", + " #print(predictor.summary())\n", + " return predictor\n", + "\n", + "def error_rate_for_model(test_model, train_set, test_set, infer=False, infer_steps=3, infer_alpha=0.1, infer_subsample=0.1):\n", + " \"\"\"Report error rate on test_doc sentiments, using supplied model and train_docs\"\"\"\n", + "\n", + " train_targets, train_regressors = zip(*[(doc.sentiment, test_model.docvecs[doc.tags[0]]) for doc in train_set])\n", + " train_regressors = sm.add_constant(train_regressors)\n", + " predictor = logistic_predictor_from_data(train_targets, train_regressors)\n", + "\n", + " test_data = test_set\n", + " if infer:\n", + " if infer_subsample < 1.0:\n", + " test_data = sample(test_data, int(infer_subsample * len(test_data)))\n", + " test_regressors = [test_model.infer_vector(doc.words, steps=infer_steps, alpha=infer_alpha) for doc in test_data]\n", + " else:\n", + " test_regressors = [test_model.docvecs[doc.tags[0]] for doc in test_docs]\n", + " test_regressors = sm.add_constant(test_regressors)\n", + " \n", + " # predict & evaluate\n", + " test_predictions = predictor.predict(test_regressors)\n", + " corrects = sum(np.rint(test_predictions) == [doc.sentiment for doc in test_data])\n", + " errors = len(test_predictions) - corrects\n", + " error_rate = float(errors) / len(test_predictions)\n", + " return (error_rate, errors, len(test_predictions), predictor)\n" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Bulk Training" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using explicit multiple-pass, alpha-reduction approach as sketched in [gensim doc2vec blog post](http://radimrehurek.com/2014/12/doc2vec-tutorial/) \u2013 with added shuffling of corpus on each pass.\n", + "\n", + "Note that vector training is occurring on *all* documents of the dataset, which includes all TRAIN/TEST/DEV docs.\n", + "\n", + "Evaluation of each model's sentiment-predictive power is repeated after each pass, as an error rate (lower is better), to see the rates-of-relative-improvement. The base numbers reuse the TRAIN and TEST vectors stored in the models for the logistic regression, while the _inferred_ results use newly-inferred TEST vectors. \n", + "\n", + "(On a 4-core 2.6Ghz Intel Core i7, these 20 passes training and evaluating 3 main models takes about an hour.)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from collections import defaultdict\n", + "best_error = defaultdict(lambda :1.0) # to selectively-print only best errors achieved" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from random import shuffle\n", + "import datetime\n", + "\n", + "alpha, min_alpha, passes = (0.025, 0.001, 20)\n", + "alpha_delta = (alpha - min_alpha) / passes\n", + "\n", + "print(\"START %s\" % datetime.datetime.now())\n", + "\n", + "for epoch in range(passes):\n", + " shuffle(doc_list) # shuffling gets best results\n", + " \n", + " for name, train_model in models_by_name.items():\n", + " # train\n", + " duration = 'na'\n", + " train_model.alpha, train_model.min_alpha = alpha, alpha\n", + " with elapsed_timer() as elapsed:\n", + " train_model.train(doc_list)\n", + " duration = '%.1f' % elapsed()\n", + " \n", + " # evaluate\n", + " eval_duration = ''\n", + " with elapsed_timer() as eval_elapsed:\n", + " err, err_count, test_count, predictor = error_rate_for_model(train_model, train_docs, test_docs)\n", + " eval_duration = '%.1f' % eval_elapsed()\n", + " best_indicator = ' '\n", + " if err <= best_error[name]:\n", + " best_error[name] = err\n", + " best_indicator = '*' \n", + " print(\"%s%f : %i passes : %s %ss %ss\" % (best_indicator, err, epoch + 1, name, duration, eval_duration))\n", + "\n", + " if (epoch % 5) == 0:\n", + " eval_duration = ''\n", + " with elapsed_timer() as eval_elapsed:\n", + " infer_err, err_count, test_count, predictor = error_rate_for_model(train_model, train_docs, test_docs, infer=True)\n", + " eval_duration = '%.1f' % eval_elapsed()\n", + " best_indicator = ' '\n", + " if infer_err < best_error[name + '_inferred']:\n", + " best_error[name + '_inferred'] = infer_err\n", + " best_indicator = '*'\n", + " print(\"%s%f : %i passes : %s %ss %ss\" % (best_indicator, infer_err, epoch + 1, name + '_inferred', duration, eval_duration))\n", + "\n", + " print('completed pass %i at alpha %f' % (epoch + 1, alpha))\n", + " alpha -= alpha_delta\n", + " \n", + "print(\"END %s\" % str(datetime.datetime.now()))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "START 2015-06-28 20:34:29.500839\n", + "*0.417080 : 1 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 84.5s 1.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.363200 : 1 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8)_inferred 84.5s 14.9s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.219520 : 1 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.0s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.184000 : 1 passes : Doc2Vec(dbow,d100,n5,mc2,t8)_inferred 19.0s 4.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.277080 : 1 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.0s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.230800 : 1 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8)_inferred 35.0s 6.4s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.207840 : 1 passes : dbow+dmm 0.0s 1.5s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.185200 : 1 passes : dbow+dmm_inferred 0.0s 11.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.220720 : 1 passes : dbow+dmc 0.0s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.189200 : 1 passes : dbow+dmc_inferred 0.0s 19.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 1 at alpha 0.025000\n", + "*0.357120 : 2 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 73.1s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.144360 : 2 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.8s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.225640 : 2 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 36.2s 1.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.141160 : 2 passes : dbow+dmm 0.0s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.144800 : 2 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 2 at alpha 0.023800\n", + "*0.326840 : 3 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 73.6s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.125880 : 3 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 20.1s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.202680 : 3 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 36.0s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.123280 : 3 passes : dbow+dmm 0.0s 1.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.126040 : 3 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 3 at alpha 0.022600\n", + "*0.302360 : 4 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 72.6s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.113640 : 4 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.9s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.189880 : 4 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.8s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.114200 : 4 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.115640 : 4 passes : dbow+dmc 0.0s 1.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 4 at alpha 0.021400\n", + "*0.281480 : 5 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 72.7s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.109720 : 5 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 21.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.181360 : 5 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 37.8s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.109760 : 5 passes : dbow+dmm 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.110400 : 5 passes : dbow+dmc 0.0s 1.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 5 at alpha 0.020200\n", + "*0.264640 : 6 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 72.0s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.292000 : 6 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8)_inferred 72.0s 13.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.107440 : 6 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 21.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.116000 : 6 passes : Doc2Vec(dbow,d100,n5,mc2,t8)_inferred 21.6s 4.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.176040 : 6 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 37.4s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.213600 : 6 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8)_inferred 37.4s 6.4s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.107000 : 6 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.108000 : 6 passes : dbow+dmm_inferred 0.0s 11.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.107880 : 6 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.124400 : 6 passes : dbow+dmc_inferred 0.0s 18.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 6 at alpha 0.019000\n", + "*0.254200 : 7 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 65.7s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.106720 : 7 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.172880 : 7 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.106080 : 7 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.106320 : 7 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 7 at alpha 0.017800\n", + "*0.245880 : 8 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 68.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.104920 : 8 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 20.0s 1.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.171000 : 8 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.4s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.104760 : 8 passes : dbow+dmm 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.105600 : 8 passes : dbow+dmc 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 8 at alpha 0.016600\n", + "*0.238400 : 9 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 66.1s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.104520 : 9 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 21.2s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.167600 : 9 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 37.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.103680 : 9 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.103480 : 9 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 9 at alpha 0.015400\n", + "*0.232160 : 10 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 69.0s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.103680 : 10 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 21.8s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.166000 : 10 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.4s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101920 : 10 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.103560 : 10 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 10 at alpha 0.014200\n", + "*0.227760 : 11 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 66.4s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.242400 : 11 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8)_inferred 66.4s 13.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.102160 : 11 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.7s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.113200 : 11 passes : Doc2Vec(dbow,d100,n5,mc2,t8)_inferred 19.7s 5.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.163480 : 11 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.4s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.208800 : 11 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8)_inferred 35.4s 6.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101560 : 11 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.102000 : 11 passes : dbow+dmm_inferred 0.0s 11.4s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101920 : 11 passes : dbow+dmc 0.0s 1.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.109600 : 11 passes : dbow+dmc_inferred 0.0s 17.4s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 11 at alpha 0.013000\n", + "*0.225960 : 12 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 61.8s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101720 : 12 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 20.2s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.163000 : 12 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.100840 : 12 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101920 : 12 passes : dbow+dmc 0.0s 1.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 12 at alpha 0.011800\n", + "*0.222360 : 13 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 65.2s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.103120 : 13 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 20.0s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.161960 : 13 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.2s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101640 : 13 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.102600 : 13 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 13 at alpha 0.010600\n", + "*0.220960 : 14 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 65.3s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.102920 : 14 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.9s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.160160 : 14 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 36.0s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101720 : 14 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.102560 : 14 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 14 at alpha 0.009400\n", + "*0.219400 : 15 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 64.0s 1.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101440 : 15 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.160640 : 15 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 38.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.100160 : 15 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101880 : 15 passes : dbow+dmc 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 15 at alpha 0.008200\n", + "*0.216880 : 16 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 64.1s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.232400 : 16 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8)_inferred 64.1s 12.8s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101760 : 16 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.1s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.111600 : 16 passes : Doc2Vec(dbow,d100,n5,mc2,t8)_inferred 19.1s 4.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.159800 : 16 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 34.9s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.184000 : 16 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8)_inferred 34.9s 6.5s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.100640 : 16 passes : dbow+dmm 0.0s 1.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.094800 : 16 passes : dbow+dmm_inferred 0.0s 11.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101320 : 16 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.109600 : 16 passes : dbow+dmc_inferred 0.0s 17.5s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 16 at alpha 0.007000\n", + " 0.217160 : 17 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 58.6s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101760 : 17 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.159640 : 17 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 37.0s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.100760 : 17 passes : dbow+dmm 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101480 : 17 passes : dbow+dmc 0.0s 1.3s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 17 at alpha 0.005800\n", + "*0.216080 : 18 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 60.7s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101520 : 18 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.6s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.158760 : 18 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 34.9s 1.0s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.100800 : 18 passes : dbow+dmm 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101760 : 18 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 18 at alpha 0.004600\n", + "*0.215560 : 19 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 62.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.101000 : 19 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 20.6s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.159080 : 19 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 35.9s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "*0.099920 : 19 passes : dbow+dmm 0.0s 1.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.102280 : 19 passes : dbow+dmc 0.0s 1.2s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 19 at alpha 0.003400\n", + "*0.215160 : 20 passes : Doc2Vec(dm/c,d100,n5,w5,mc2,t8) 58.3s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.101360 : 20 passes : Doc2Vec(dbow,d100,n5,mc2,t8) 19.5s 0.7s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.158920 : 20 passes : Doc2Vec(dm/m,d100,n5,w10,mc2,t8) 33.6s 0.6s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.100480 : 20 passes : dbow+dmm 0.0s 1.5s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + " 0.102160 : 20 passes : dbow+dmc 0.0s 1.1s" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "completed pass 20 at alpha 0.002200\n", + "END 2015-06-28 21:20:48.994706\n" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Achieved Sentiment-Prediction Accuracy" + ] + }, + { + "cell_type": "code", + "collapsed": true, + "input": [ + "# print best error rates achieved\n", + "for rate, name in sorted((rate, name) for name, rate in best_error.items()):\n", + " print(\"%f %s\" % (rate, name))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0.094800 dbow+dmm_inferred\n", + "0.099920 dbow+dmm\n", + "0.101000 Doc2Vec(dbow,d100,n5,mc2,t8)\n", + "0.101320 dbow+dmc\n", + "0.109600 dbow+dmc_inferred\n", + "0.111600 Doc2Vec(dbow,d100,n5,mc2,t8)_inferred\n", + "0.158760 Doc2Vec(dm/m,d100,n5,w10,mc2,t8)\n", + "0.184000 Doc2Vec(dm/m,d100,n5,w10,mc2,t8)_inferred\n", + "0.215160 Doc2Vec(dm/c,d100,n5,w5,mc2,t8)\n", + "0.232400 Doc2Vec(dm/c,d100,n5,w5,mc2,t8)_inferred\n" + ] + } + ], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In my testing, unlike the paper's report, DBOW performs best. Concatenating vectors from different models only offers a small predictive improvement. The best results I've seen are still just under 10% error rate, still a ways from the paper's 7.42%.\n" + ] + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Examining Results" + ] + }, + { + "cell_type": "heading", + "level": 3, + "metadata": {}, + "source": [ + "Are inferred vectors close to the precalculated ones?" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "doc_id = np.random.randint(simple_models[0].docvecs.count) # pick random doc; re-run cell for more examples\n", + "print('for doc %d...' % doc_id)\n", + "for model in simple_models:\n", + " inferred_docvec = model.infer_vector(alldocs[doc_id].words)\n", + " print('%s:\\n %s' % (model, model.docvecs.most_similar([inferred_docvec], topn=3)))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "for doc 25430...\n", + "Doc2Vec(dm/c,d100,n5,w5,mc2,t8):\n", + " [(25430, 0.6583491563796997), (27314, 0.4142411947250366), (16479, 0.40846431255340576)]" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "Doc2Vec(dbow,d100,n5,mc2,t8):\n", + " [(25430, 0.9325973987579346), (49281, 0.5766637921333313), (79679, 0.5634804964065552)]" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "Doc2Vec(dm/m,d100,n5,w10,mc2,t8):\n", + " [(25430, 0.7970066666603088), (97818, 0.6925815343856812), (230, 0.690807580947876)]" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n" + ] + } + ], + "prompt_number": 13 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(Yes, here the stored vector from 20 epochs of training is usually one of the closest to a freshly-inferred vector for the same words. Note the defaults for inference are very abbreviated \u2013 just 3 steps starting at a high alpha \u2013 and likely need tuning for other applications.)" + ] + }, + { + "cell_type": "heading", + "level": 3, + "metadata": {}, + "source": [ + "Do close documents seem more related than distant ones?" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import random\n", + "\n", + "doc_id = np.random.randint(simple_models[0].docvecs.count) # pick random doc, re-run cell for more examples\n", + "model = random.choice(simple_models) # and a random model\n", + "sims = model.docvecs.most_similar(doc_id, topn=model.docvecs.count) # get *all* similar documents\n", + "print('TARGET (%d): \u00ab%s\u00bb\\n' % (doc_id, ' '.join(alldocs[doc_id].words)))\n", + "print('SIMILAR/DISSIMILAR DOCS PER MODEL %s:\\n' % model)\n", + "for label, index in [('MOST', 0), ('MEDIAN', len(sims)//2), ('LEAST', len(sims) - 1)]:\n", + " print('%s %s: \u00ab%s\u00bb\\n' % (label, sims[index], ' '.join(alldocs[sims[index][0]].words)))\n" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "TARGET (72927): \u00abthis is one of the best films of this year . for a year that was fueled by controversy and crap , it was nice to finally see a film that had a true heart to it . from the opening scene to the end , i was so moved by the love that will smith has for his son . basically , if you see this movie and walk out of it feeling nothing , there is something that is very wrong with you . loved this movie , it's the perfect movie to end the year with . the best part was after the movie , my friends and i all got up and realized that this movie had actually made the four of us tear up ! it's an amazing film and if will smith doesn't get at least an oscar nom , then the oscars will just suck . in fact will smith should actually just win an oscar for this role . ! ! ! i loved this movie ! ! ! ! everybody needs to see especially the people in this world that take everything for granted , watch this movie , it will change you !\u00bb\n", + "\n", + "SIMILAR/DISSIMILAR DOCS PER MODEL Doc2Vec(dm/m,d100,n5,w10,mc2,t8):\n", + "\n", + "MOST (2046, 0.7372332215309143): \u00abi thought this movie would be dumb , but i really liked it . people i know hate it because spirit was the only horse that talked . well , so what ? the songs were good , and the horses didn't need to talk to seem human . i wouldn't care to own the movie , and i would love to see it again . 8/10\u00bb\n", + "\n", + "MEDIAN (6999, 0.4129640758037567): \u00abokay , the recent history of star trek has not been good . the next generation faded in its last few seasons , ds9 boldly stayed where no one had stayed before , and voyager started very bad and never really lived up to its promise . so , when they announced a new star trek series , i did not have high expectations . and , the first episode , broken bow , did have some problems . but , overall it was solid trek material and a good romp . i'll get the nits out of the way first . the opening theme is dull and i don't look forward to sitting through it regularly , but that's what remotes are for . what was really bad was the completely gratuitous lotion rubbing scene that just about drove my wife out of the room . they need to cut that nonsense out . but , the plot was strong and moved along well . the characters , though still new , seem to be well rounded and not always what you would expect . the vulcans are clearly being presented very differently than before , with a slightly ominous theme . i particularly liked the linguist , who is the first star trek character to not be able to stand proud in the face of death , but rather has to deal with her phobias and fears . they seemed to stay true to trek lore , something that has been a significant problem in past series , though they have plenty of time to bring us things like shooting through shields , the instant invention of technology that can fix anything , and the inevitable plethora of time-travel stories . anyone want to start a pool on how long before the borg show up ? all in all , the series has enormous potential . they are seeing the universe with fresh eyes . we have the chance to learn how things got the way they were in the later series . how did the klingons go from just insulting to war ? how did we meet the romulans ? how did the federation form and just who put earth in charge . why is the prime directive so important ? if they address these things rather than spitting out time travel episodes , this will be an interesting series . my favorite line : zephram cochran saying \" where no man has gone before \" ( not \" no one \" )\u00bb\n", + "\n", + "LEAST (16617, 0.015464222989976406): \u00abi saw this movie during a tolkien-themed interim class during my sophomore year of college . i was seated unfortunately close to the screen and my professor chose me to serve as a whipping boy- everyone else was laughing , but they weren't within constant eyesight . let's get it out of the way : the peter jackson 'lord of the rings' films do owe something to the bakshi film . in jackson's version of the fellowship of the ring , for instance , the scene in which the black riders assault the empty inn beds is almost a complete carbon copy of the scene in bakshi's film , shot by shot . you could call this plagiarism or homage , depending on your agenda . i'm sure the similarities don't stop there . i'm not going to do any research to find out what they are , because that would imply i have some mote of respect for this film . i'm sure others have outlined the similarities- look around . this movie is a complete train wreck in every sense of the metaphor , and many , many people died in the accident . i've decided to list what i can remember in a more or less chronological fashion- if i've left out anything else that offended me it's because i'm completely overwhelmed , confronted with a wealth of failure ( and , at high points , mediocrity ) . *due to heavy use of rotoscoping , gandalf is no longer a gentle , wise wizard but a wildly flailing prophet of doom ( whose hat inexplicably changes color once or twice during the course of the film ) . *saruman the white is sometimes referred to as 'aruman' during the film , without explanation . he wears purple and red for some mysterious reason . *sam is flat out hideous . the portrayal of his friendship with frodo is strangely childlike and unsatisfying . yes , hobbits are small like children , but they are not children . *merry and pippin are never introduced--they simply appear during a scene change with a one-sentence explanation . the film is filled with sloppy editing like this . *frodo , sam , pippin and merry are singing merrily as they skip through along the road . one of the hobbits procures a lute at least twice as large as he is from behind his back--which was not visible before--and begins strumming in typical fantasy bard fashion as they all break into \" la-la-la \" s . awful . *aragorn , apparently , is a native american dressed in an extremely stereotypical fantasy tunic ( no pants ) , complete with huge , square pilgrim belt buckle . he is arguably the worst swordsman in the entire movie--oftentimes he gets one wobbly swing in before being knocked flat on his ass . *the black riders appear more like lepers than menacing instruments of evil . they limp everywhere they go at a painfully slow pace . this is disturbing to be sure , but not frightening . *the scene before the black riders attempt to cross the ford of bruinen ( in which they stare at frodo , who is on the other side on horseback ) goes on forever , during which time the riders rear their horses in a vaguely threatening manner and . . . do nothing else . the scene was probably intended to illustrate frodo's hallucinatory decline as he succumbs to his wound . it turns out to be more plodding than anything else . *gimli the dwarf is just as tall as legolas the elf . he's a dwarf . there is simply no excuse for that . he also looks like a bastardized david the gnome . it's a crude but accurate description . *boromir appears to have pilfered elmer fudd's golden viking armor from that bugs bunny opera episode . he looks ridiculous . *despite the similarity to tolkien's illustration , the balrog is howl inducing and the least-threatening villain in the entire film . it looks like someone wearing pink bedroom slippers , and it's barely taller than gandalf . \" purists \" may prefer this balrog , but i'll take jackson's version any day . *the battle scenes are awkward and embarrassing . almost none of the characters display any level of competency with their armaments . i'm not asking for action-packed scenes like those in jackson's film , but they are supposed to be fighting . *treebeard makes a very short appearance , and i was sorry he bothered to show up at all . watch the film , you'll see what i mean . alright , now for the good parts of the film . *some of the voice acting is pretty good . it isn't that aragorn sounds bad , he just looks kind of like the jolly green giant . *galadriel is somewhat interesting in this portrayal ; like tom bombadil , she seems immune to the ring's powers of temptation , and her voice actress isn't horrible either . *boromir's death isn't as heart wrenching as in jackson's portrayal of the same scene , but it's still appropriately dramatic ( and more true to his death in the book , though i don't believe jackson made a mistake shooting it the way he did ) . *as my professor pointed out ( between whispered threats ) , the orcs ( mainly at helm's deep , if i'm correct ) resemble the war-ravaged corpses of soldiers , a political statement that works pretty well if you realize what's being attempted . *while this isn't really a positive point about the film , bakshi can't be blamed for the majority of the failures in this movie , or so i've been told--the project was on a tight budget , and late in its production he lost creative control to some of the higher-ups ( who i'm sure hadn't read the books ) . let me be clear : i respect bakshi for even attempting something of this magnitude . i simply have a hard time believing he was happy with the final product . overall , i cannot in any way recommend this blasphemous adaptation of tolkien's classic trilogy even for laughs , unless you've already read the books and have your own visualizations of the characters , places and events . i'm sure somebody , somewhere , will pick a copy of this up in confusion ; if you do , keep an open mind and glean what good you can from it .\u00bb\n", + "\n" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(Somewhat, in terms of reviewer tone, movie genre, etc... the MOST cosine-similar docs usually seem more like the TARGET than the MEDIAN or LEAST.)" + ] + }, + { + "cell_type": "heading", + "level": 3, + "metadata": {}, + "source": [ + "Do the word vectors show useful similarities?" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "word_models = simple_models[:]" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 15 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import random\n", + "from IPython.display import HTML\n", + "# pick a random word with a suitable number of occurences\n", + "while True:\n", + " word = random.choice(word_models[0].index2word)\n", + " if word_models[0].vocab[word].count > 10:\n", + " break\n", + "# or just pick a word from the relevant domain:\n", + "word = 'comedy/drama'\n", + "similars_per_model = [str(model.most_similar(word, topn=20)).replace('), ','),
\\n') for model in word_models]\n", + "similar_table = (\"
\" +\n", + " \"\".join([str(model) for model in word_models]) + \n", + " \"
\" +\n", + " \"\".join(similars_per_model) +\n", + " \"
\")\n", + "print(\"most similar words for '%s' (%d occurences)\" % (word, simple_models[0].vocab[word].count))\n", + "HTML(similar_table)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "most similar words for 'comedy/drama' (38 occurences)\n" + ] + }, + { + "html": [ + "
Doc2Vec(dm/c,d100,n5,w5,mc2,t8)Doc2Vec(dbow,d100,n5,mc2,t8)Doc2Vec(dm/m,d100,n5,w10,mc2,t8)
[('comedy', 0.7255545258522034),
\n", + "('thriller', 0.6946465969085693),
\n", + "('drama', 0.6763534545898438),
\n", + "('romance', 0.6251884698867798),
\n", + "('dramedy', 0.6217159032821655),
\n", + "('melodrama', 0.6156137585639954),
\n", + "('adventure', 0.6091135740280151),
\n", + "('farce', 0.6034293174743652),
\n", + "('chiller', 0.5948368906974792),
\n", + "('romantic-comedy', 0.5876704454421997),
\n", + "('fantasy', 0.5863304138183594),
\n", + "('mystery/comedy', 0.577541708946228),
\n", + "('whodunit', 0.572147011756897),
\n", + "('biopic', 0.5679721832275391),
\n", + "('thriller/drama', 0.5630226731300354),
\n", + "('sitcom', 0.5574496984481812),
\n", + "('slash-fest', 0.5573585033416748),
\n", + "('mystery', 0.5542301535606384),
\n", + "('potboiler', 0.5519827604293823),
\n", + "('mockumentary', 0.5490710139274597)]		[('1000%', 0.42290645837783813),
\n", + "(\"gymnast's\", 0.4180164337158203),
\n", + "('hollywoodland', 0.3898555636405945),
\n", + "('cultures', 0.3857914209365845),
\n", + "('hooda', 0.3851744532585144),
\n", + "('cites', 0.38047513365745544),
\n", + "(\"78's\", 0.3792475461959839),
\n", + "(\"dormael's\", 0.3775535225868225),
\n", + "('jokester', 0.3725704252719879),
\n", + "('impelled', 0.36853262782096863),
\n", + "('lia', 0.3684236407279968),
\n", + "('snivelling', 0.3683513104915619),
\n", + "('astral', 0.36715900897979736),
\n", + "('euro-exploitation', 0.35853487253189087),
\n", + "(\"serra's\", 0.3578598201274872),
\n", + "('down-on-their-luck', 0.3576606214046478),
\n", + "('rowles', 0.3567575514316559),
\n", + "('romantica', 0.3549702763557434),
\n", + "('bonham-carter', 0.354231059551239),
\n", + "('1877', 0.3541453182697296)]		[('comedy-drama', 0.6274900436401367),
\n", + "('comedy', 0.5986765623092651),
\n", + "('thriller', 0.5765297412872314),
\n", + "('road-movie', 0.5615973472595215),
\n", + "('dramedy', 0.5580120086669922),
\n", + "('time-killer', 0.5497636795043945),
\n", + "('potboiler', 0.5456510782241821),
\n", + "('comedy/', 0.5439876317977905),
\n", + "('actioner', 0.5423712134361267),
\n", + "('diversion', 0.541743278503418),
\n", + "('romcom', 0.5402226448059082),
\n", + "('rom-com', 0.5358527302742004),
\n", + "('drama', 0.5320745706558228),
\n", + "('chiller', 0.5229591727256775),
\n", + "('romp', 0.5228806734085083),
\n", + "('horror/comedy', 0.5219299793243408),
\n", + "('weeper', 0.5195824503898621),
\n", + "('mockumentary', 0.5149033069610596),
\n", + "('camp-fest', 0.5122634768486023),
\n", + "('mystery/comedy', 0.5020694732666016)]
" + ], + "metadata": {}, + "output_type": "pyout", + "prompt_number": 17, + "text": [ + "" + ] + } + ], + "prompt_number": 17 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Do the DBOW words look meaningless? That's because the gensim DBOW model doesn't train word vectors \u2013 they remain at their random initialized values \u2013 unless you ask with the `dbow_words=1` initialization parameter. Concurrent word-training slows DBOW mode significantly, and offers little improvement (and sometimes a little worsening) of the error rate on this IMDB sentiment-prediction task. \n", + "\n", + "Words from DM models tend to show meaningfully similar words when there are many examples in the training data (as with 'plot' or 'actor'). (All DM modes inherently involve word vector training concurrent with doc vector training.)" + ] + }, + { + "cell_type": "heading", + "level": 3, + "metadata": {}, + "source": [ + "Are the word vectors from this dataset any good at analogies?" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# assuming something like\n", + "# https://word2vec.googlecode.com/svn/trunk/questions-words.txt \n", + "# is in local directory\n", + "# note: this takes many minutes\n", + "for model in word_models:\n", + " sections = model.accuracy('questions-words.txt')\n", + " correct, incorrect = (len(sum((s['correct'] for s in sections), [])), len(sum((s['incorrect'] for s in sections),[])))\n", + " print('%s: %0.2f%% correct (%d of %d)' % (model, float(correct*100)/(correct+incorrect), correct, correct+incorrect))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Doc2Vec(dm/c,d100,n5,w5,mc2,t8): 28.70% correct (5746 of 20024)\n", + "Doc2Vec(dbow,d100,n5,mc2,t8): 0.01% correct (2 of 20024)" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "Doc2Vec(dm/m,d100,n5,w10,mc2,t8): 27.24% correct (5454 of 20024)" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n" + ] + } + ], + "prompt_number": 26 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even though this is a tiny, domain-specific dataset, it shows some meager capability on the general word analogies \u2013 at least for the DM/concat and DM/mean models which actually train word vectors. (The untrained random-initialized words of the DBOW model of course fail miserably.)" + ] + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Slop" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "This cell left intentionally erroneous. " + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To mix the Google dataset (if locally available) into the word tests..." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from gensim.models import Word2Vec\n", + "w2v_g100b = Word2Vec.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)\n", + "w2v_g100b.compact_name = 'w2v_g100b'\n", + "word_models.append(w2v_g100b)" + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To get copious logging output from above steps..." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import logging\n", + "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n", + "rootLogger = logging.getLogger()\n", + "rootLogger.setLevel(logging.INFO)" + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To auto-reload python code while developing..." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%load_ext autoreload\n", + "%autoreload 2" + ], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/gensim/models/doc2vec.py b/gensim/models/doc2vec.py index 97696b8974..895546b107 100644 --- a/gensim/models/doc2vec.py +++ b/gensim/models/doc2vec.py @@ -14,7 +14,7 @@ Initialize a model with e.g.:: ->>> model = Doc2Vec(sentences, size=100, window=8, min_count=5, workers=4) +>>> model = Doc2Vec(documents, size=100, window=8, min_count=5, workers=4) Persist a model to disk with:: @@ -42,131 +42,446 @@ except ImportError: from Queue import Queue -from numpy import zeros, random, sum as np_sum -from six import string_types +from collections import namedtuple + +from numpy import zeros, random, sum as np_sum, add as np_add, concatenate, \ + repeat as np_repeat, array, float32 as REAL, empty, ones, memmap as np_memmap, \ + sqrt, newaxis, ndarray, dot, argsort, vstack logger = logging.getLogger(__name__) -from gensim import utils # utility fnc for pickling, common scipy operations etc -from gensim.models.word2vec import Word2Vec, Vocab, train_cbow_pair, train_sg_pair +from gensim import utils, matutils # utility fnc for pickling, common scipy operations etc +from gensim.models.word2vec import Word2Vec, Vocab, train_cbow_pair, train_sg_pair, train_sentence_sg +from six.moves import xrange +from six import string_types, integer_types try: - from gensim.models.doc2vec_inner import train_sentence_dbow, train_sentence_dm, FAST_VERSION + from gensim.models.doc2vec_inner import train_document_dbow, train_document_dm, train_document_dm_concat,\ + FAST_VERSION except: # failed... fall back to plain numpy (20-80x slower training than the above) FAST_VERSION = -1 - def train_sentence_dbow(model, sentence, lbls, alpha, work=None, train_words=True, train_lbls=True): + def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, + train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): """ - Update distributed bag of words model by training on a single sentence. + Update distributed bag of words model ("PV-DBOW") by training on a single document. + + Called internally from `Doc2Vec.train()` and `Doc2Vec.infer_vector()`. + + The document is provided as `word_vocabs`, a list of Vocab objects which provide + indexes into the word_vector array, and `doctag_indexes`, which provide indexes + int the doctag_vectors array. (See `_prepare_items()`.) - The sentence is a list of Vocab objects (or None, where the corresponding - word is not in the vocabulary. Called internally from `Doc2Vec.train()`. + If `train_words` is True, simultaneously train word-to-word (not just doc-to-word) + examples, exactly as per Word2Vec skip-gram training. (Without this option, + word vectors are neither consulted nor updated during DBOW doc vector training.) + + Any of `learn_doctags', `learn_words`, and `learn_hidden` may be set False to + prevent learning-updates to those respective model weights, as if using the + (partially-)frozen model to infer other compatible vectors. This is the non-optimized, Python version. If you have cython installed, gensim will use the optimized version from doc2vec_inner instead. """ - neg_labels = [] - if model.negative: - # precompute negative labels - neg_labels = zeros(model.negative + 1) - neg_labels[0] = 1.0 - - for label in lbls: - if label is None: - continue # OOV word in the input sentence => skip - for word in sentence: + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + + if train_words and learn_words: + train_sentence_sg(model, word_vocabs, alpha, work) # TODO: adapt for word_vectors/word_locks + for doctag_index in doctag_indexes: + for word in word_vocabs: if word is None: - continue # OOV word in the input sentence => skip - train_sg_pair(model, word, label, alpha, neg_labels, train_words, train_lbls) + continue # OOV word in the input document => skip + train_sg_pair(model, word, doctag_index, alpha, learn_vectors=learn_doctags, + learn_hidden=learn_hidden, context_vectors=doctag_vectors, + context_locks=doctag_locks) - return len([word for word in sentence if word is not None]) + return len([word for word in word_vocabs if word is not None]) - def train_sentence_dm(model, sentence, lbls, alpha, work=None, neu1=None, train_words=True, train_lbls=True): + def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): """ - Update distributed memory model by training on a single sentence. + Update distributed memory model ("PV-DM") by training on a single document. + + Called internally from `Doc2Vec.train()` and `Doc2Vec.infer_vector()`. This + method implements the DM model with a projection (input) layer that is + either the sum or mean of the context vectors, depending on the model's + `dm_mean` configuration field. See `train_dm_concat()` for the DM model + with a concatenated input layer. - The sentence is a list of Vocab objects (or None, where the corresponding - word is not in the vocabulary. Called internally from `Doc2Vec.train()`. + The document is provided as `word_vocabs`, a list of Vocab objects which provide + indexes into the word_vector array, and `doctag_indexes`, which provide indexes + int the doctag_vectors array. (See `_prepare_items()`.) + + Any of `learn_doctags', `learn_words`, and `learn_hidden` may be set False to + prevent learning-updates to those respective model weights, as if using the + (partially-)frozen model to infer other compatible vectors. This is the non-optimized, Python version. If you have a C compiler, gensim will use the optimized version from doc2vec_inner instead. """ - lbl_indices = [lbl.index for lbl in lbls if lbl is not None] - lbl_sum = np_sum(model.syn0[lbl_indices], axis=0) - lbl_len = len(lbl_indices) - neg_labels = [] - if model.negative: - # precompute negative labels - neg_labels = zeros(model.negative + 1) - neg_labels[0] = 1. - - for pos, word in enumerate(sentence): + if word_vectors is None: + word_vectors = model.syn0 + if word_locks is None: + word_locks = model.syn0_lockf + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + + doctag_sum = np_sum(doctag_vectors[doctag_indexes], axis=0) + doctag_len = len(doctag_indexes) + + for pos, word in enumerate(word_vocabs): if word is None: - continue # OOV word in the input sentence => skip + continue # OOV word in the input document => skip reduced_window = random.randint(model.window) # `b` in the original doc2vec code start = max(0, pos - model.window + reduced_window) - window_pos = enumerate(sentence[start : pos + model.window + 1 - reduced_window], start) - word2_indices = [word2.index for pos2, word2 in window_pos if (word2 is not None and pos2 != pos)] - l1 = np_sum(model.syn0[word2_indices], axis=0) + lbl_sum # 1 x layer1_size - if word2_indices and model.cbow_mean: - l1 /= (len(word2_indices) + lbl_len) - neu1e = train_cbow_pair(model, word, word2_indices, l1, alpha, neg_labels, train_words, train_words) - if train_lbls: - model.syn0[lbl_indices] += neu1e + window_pos = enumerate(word_vocabs[start : pos + model.window + 1 - reduced_window], start) + word2_indexes = [word2.index for pos2, word2 in window_pos if (word2 is not None and pos2 != pos)] + l1 = np_sum(word_vectors[word2_indexes], axis=0) + doctag_sum # 1 x layer1_size + if word2_indexes and model.cbow_mean: + l1 /= (len(word2_indexes) + doctag_len) + neu1e = train_cbow_pair(model, word, word2_indexes, l1, alpha, learn_vectors=False, learn_hidden=True) + if word2_indexes and not model.cbow_mean: + neu1e /= (len(word2_indexes) + doctag_len) + if learn_doctags: + doctag_vectors[doctag_indexes] += \ + neu1e * np_repeat(doctag_locks[doctag_indexes],model.vector_size).reshape(-1,model.vector_size) + if learn_words: + word_vectors[word2_indexes] += \ + neu1e * np_repeat(word_locks[word2_indexes],model.vector_size).reshape(-1,model.vector_size) + + return len([word for word in word_vocabs if word is not None]) + + + def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + """ + Update distributed memory model ("PV-DM") by training on a single document, using a + concatenation of the context window word vectors (rather than a sum or average). - return len([word for word in sentence if word is not None]) + Called internally from `Doc2Vec.train()` and `Doc2Vec.infer_vector()`. + The document is provided as `word_vocabs`, a list of Vocab objects which provide + indexes into the word_vector array, and `doctag_indexes`, which provide indexes + int the doctag_vectors array. (See `_prepare_items()`.) -class LabeledSentence(object): + Any of `learn_doctags', `learn_words`, and `learn_hidden` may be set False to + prevent learning-updates to those respective model weights, as if using the + (partially-)frozen model to infer other compatible vectors. + + This is the non-optimized, Python version. If you have a C compiler, gensim + will use the optimized version from doc2vec_inner instead. + + """ + if word_vectors is None: + word_vectors = model.syn0 + if word_locks is None: + word_locks = model.syn0_lockf + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + + doctag_len = len(doctag_indexes) + if doctag_len != model.dm_tag_count: + return 0 # skip doc without expected doctag(s) + + null_word = model.vocab['\0'] + pre_pad_count = model.window + post_pad_count = model.window + padded_document_indexes = ( + (pre_pad_count * [null_word.index]) # pre-padding + + [word.index for word in word_vocabs if word is not None] # elide out-of-Vocabulary words + + (post_pad_count * [null_word.index]) # post-padding + ) + + for pos in range(pre_pad_count, len(padded_document_indexes) - post_pad_count): + word_context_indexes = ( + padded_document_indexes[pos - pre_pad_count : pos] # preceding words + + padded_document_indexes[pos + 1 : pos + 1 + post_pad_count] # following words + ) + word_context_len = len(word_context_indexes) + predict_word = model.vocab[model.index2word[padded_document_indexes[pos]]] + # numpy advanced-indexing copies; concatenate, flatten to 1d + l1 = concatenate((doctag_vectors[doctag_indexes], word_vectors[word_context_indexes])).ravel() + neu1e = train_cbow_pair(model, predict_word, None, l1, alpha, learn_hidden=learn_hidden, learn_vectors=False) + + # filter by locks and shape for addition to source vectors + e_locks = concatenate((doctag_locks[doctag_indexes], word_locks[word_context_indexes])) + neu1e_r = (neu1e.reshape(-1,model.vector_size) + * np_repeat(e_locks,model.vector_size).reshape(-1,model.vector_size)) + + if learn_doctags: + np_add.at(doctag_vectors, doctag_indexes, neu1e_r[:doctag_len]) + if learn_words: + np_add.at(word_vectors, word_context_indexes, neu1e_r[doctag_len:]) + + return len(padded_document_indexes) - pre_pad_count - post_pad_count + + +class TaggedDocument(namedtuple('TaggedDocument','words tags')): """ - A single labeled sentence = text item. + A single document, made up of `words` (a list of unicode string tokens) + and `tags` (a list of tokens). Tags may be one or more unicode string + tokens, but typical practice (which will also be most memory-efficient) is + for the tags list to include a unique integer id as the only tag. + Replaces "sentence as a list of words" from Word2Vec. """ - def __init__(self, words, labels): + def __str__(self): + return '%s(%s, %s)' % (self.__class__.__name__, self.words, self.tags) + + +class DocvecsArray(utils.SaveLoad): + """ + Default storage of doc vectors during/after training, in a numpy array. + + As the 'docvecs' property of a Doc2Vec model, allows access and + comparison of document vectors. + + >>> docvec = d2v_model.docvecs[99] + >>> docvec = d2v_model.docvecs['SENT_99'] # if string tag used in training + >>> sims = d2v_model.docvecs.most_similar(99) + >>> sims = d2v_model.docvecs.most_similar('SENT_99')) + >>> sims = d2v_model.docvecs.most_similar(docvec)) + + If only plain int tags are presented during training, the dict (of + string tag -> index) and list (of index -> string tag) stay empty, + saving memory. + + Supplying a mapfile_path (as by initializing a Doc2Vec model with a + 'docvecs_mapfile' value) will use a pair of memory-mapped + files as the array backing for doctag_syn0/doctag_syn0_lockf values. + + The Doc2Vec model automatically uses this class, but a future alternative + implementation, based on another persistence mechanism like LMDB, LevelDB, + or SQLite, should also be possible. + """ + def __init__(self, mapfile_path=None): + self.doctags = {} # string -> Doctag (only filled if necessary) + self.index2doctag = [] # int index -> String (only filled if necessary) + self.count = -1 + self.mapfile_path = mapfile_path + + def note_doctag(self, key, document_no, document_length): + """Note a document tag during initial corpus scan, for structure sizing.""" + if isinstance(key, int): + self.count = max(self.count, key+1) + else: + if key in self.doctags: + self.doctags[key] = self.doctags[key].repeat(document_length) + else: + self.doctags[key] = Doctag(document_no, document_length, 1) + self.index2doctag.append(key) + self.count = max(self.count, len(self.index2doctag)) + + def indexed_doctags(self, doctag_tokens): + """Return indexes and backing-arrays used in training examples.""" + return ([i for i in [self._int_index(index,-1) for index in doctag_tokens] if i > -1], + self.doctag_syn0, self.doctag_syn0_lockf, doctag_tokens) + + def trained_items(self, indexed_tuples): + """Persist any changes made to the given indexes (matching tuple previously + returned by indexed_doctags()); a no-op for this implementation""" + pass + + def _int_index(self, index, missing=None): + """Return int index for either string or int index""" + if isinstance(index, int): + return index + else: + return self.doctags[index].index if index in self.doctags else missing + + def _key_index(self, i_index, missing=None): + """Return string index for given int index, if available""" + if i_index < len(self.index2doctag): + return self.index2doctag[i_index] + else: + return i_index + + def __getitem__(self, index): + return self.doctag_syn0[self._int_index(index)] + + def __contains__(self, index): + if isinstance(index, int): + return index < self.count + else: + return index in self.doctags + + def borrow_from(self, other_docvecs): + self.count = other_docvecs.count + self.doctags = other_docvecs.doctags + self.index2doctag = other_docvecs.index2doctag + + def clear_sims(self): + self.doctag_syn0norm = None + + def reset_weights(self, model): + length = max(len(self.doctags),self.count) + if self.mapfile_path: + self.doctag_syn0 = np_memmap(self.mapfile_path+'.doctag_syn0',dtype=REAL,mode='w+',shape=(length,model.vector_size)) + self.doctag_syn0_lockf = np_memmap(self.mapfile_path+'.doctag_syn0_lockf',dtype=REAL,mode='w+',shape=(length,)) + self.doctag_syn0_lockf.fill(1.0) + else: + self.doctag_syn0 = empty((length, model.vector_size), dtype=REAL) + self.doctag_syn0_lockf = ones((length,), dtype=REAL) # zeros suppress learning + + for i in xrange(length): + # construct deterministic seed from index AND model seed + seed = "%d %s" % (model.seed, self.index2doctag[i] if len(self.index2doctag)>0 else str(i)) + self.doctag_syn0[i] = model.seeded_vector(seed) + + def init_sims(self, replace=False): """ - `words` is a list of tokens (unicode strings), - `labels` a list of text labels associated with this text - or a single string label. + Precompute L2-normalized vectors. + If `replace` is set, forget the original vectors and only keep the normalized + ones = saves lots of memory! + + Note that you **cannot continue training** after doing a replace. The model becomes + effectively read-only = you can call `most_similar`, `similarity` etc., but not `train`. + + """ + if getattr(self, 'doctag_syn0norm', None) is None or replace: + logger.info("precomputing L2-norms of doc weight vectors") + if replace: + for i in xrange(self.doctag_syn0.shape[0]): + self.doctag_syn0[i, :] /= sqrt((self.doctag_syn0[i, :] ** 2).sum(-1)) + self.doctag_syn0norm = self.doctag_syn0 + else: + self.doctag_syn0norm = (self.doctag_syn0 / sqrt((self.doctag_syn0 ** 2).sum(-1))[..., newaxis]).astype(REAL) + + def most_similar(self, positive=[], negative=[], topn=10): """ - if isinstance(labels, string_types): - labels = (labels,) - self.words = words - self.labels = labels + Find the top-N most similar docvecs known from training. Positive docs contribute + positively towards the similarity, negative docs negatively. + + This method computes cosine similarity between a simple mean of the projection + weight vectors of the given docs. Docs may be specified as vectors, integer indexes + of trained docvecs, or if the documents were originally presented with string tags, + by the corresponding tags. + """ + self.init_sims() + + if isinstance(positive, string_types + integer_types) and not negative: + # allow calls like most_similar('dog'), as a shorthand for most_similar(['dog']) + positive = [positive] + + # add weights for each doc, if not already present; default to 1.0 for positive and -1.0 for negative docs + positive = [(doc, 1.0) if isinstance(doc, string_types + (ndarray,) + integer_types) + else doc for doc in positive] + negative = [(doc, -1.0) if isinstance(doc, string_types + (ndarray,) + integer_types) + else doc for doc in negative] + + # compute the weighted average of all docs + all_docs, mean = set(), [] + for doc, weight in positive + negative: + if isinstance(doc, ndarray): + mean.append(weight * doc) + elif doc in self.doctags or doc < self.count: + mean.append(weight * self.doctag_syn0norm[self._int_index(doc)]) + all_docs.add(self._int_index(doc)) + else: + raise KeyError("doc '%s' not in trained set" % doc) + if not mean: + raise ValueError("cannot compute similarity with no input") + mean = matutils.unitvec(array(mean).mean(axis=0)).astype(REAL) + + dists = dot(self.doctag_syn0norm, mean) + if not topn: + return dists + best = argsort(dists)[::-1][:topn + len(all_docs)] + # ignore (don't return) docs from the input + result = [(self._key_index(sim), float(dists[sim])) for sim in best if sim not in all_docs] + return result[:topn] + + def doesnt_match(self, docs): + """ + Which doc from the given list doesn't go with the others? + + (TODO: Accept vectors of out-of-training-set docs, as if from inference.) + + """ + self.init_sims() + + docs = [doc for doc in docs if doc in self.doctags or 0 <= doc < self.count] # filter out unknowns + logger.debug("using docs %s" % docs) + if not docs: + raise ValueError("cannot select a doc from an empty list") + vectors = vstack(self.doctag_syn0norm[self._int_index(doc)] for doc in docs).astype(REAL) + mean = matutils.unitvec(vectors.mean(axis=0)).astype(REAL) + dists = dot(vectors, mean) + return sorted(zip(dists, docs))[0][1] + + def similarity(self, d1, d2): + """ + Compute cosine similarity between two docvecs in the trained set, specified by int index or + string tag. (TODO: Accept vectors of out-of-training-set docs, as if from inference.) + + """ + return dot(matutils.unitvec(self[d1]), matutils.unitvec(self[d2])) + + def n_similarity(self, ds1, ds2): + """ + Compute cosine similarity between two sets of docvecs from the trained set, specified by int + index or string tag. (TODO: Accept vectors of out-of-training-set docs, as if from inference.) + + """ + v1 = [self[doc] for doc in ds1] + v2 = [self[doc] for doc in ds2] + return dot(matutils.unitvec(array(v1).mean(axis=0)), matutils.unitvec(array(v2).mean(axis=0))) - def __str__(self): - return '%s(%s, %s)' % (self.__class__.__name__, self.words, self.labels) + +class Doctag(namedtuple('Doctag', 'index, word_count, doc_count')): + """A string document tag discovered during the initial vocabulary + scan. (The document-vector equivalent of a Vocab object.) + + Will not be used if all presented document tags are ints. + """ + __slots__ = () + def repeat(self, word_count): + return self._replace(word_count=self.word_count + word_count, doc_count=self.doc_count + 1) class Doc2Vec(Word2Vec): """Class for training, using and evaluating neural networks described in http://arxiv.org/pdf/1405.4053v2.pdf""" - def __init__(self, sentences=None, size=300, alpha=0.025, window=8, min_count=5, + def __init__(self, documents=None, size=300, alpha=0.025, window=8, min_count=5, sample=0, seed=1, workers=1, min_alpha=0.0001, dm=1, hs=1, negative=0, - dm_mean=0, train_words=True, train_lbls=True, **kwargs): + dbow_words=0, dm_mean=0, dm_concat=0, dm_tag_count=1, + docvecs=None, docvecs_mapfile=None, comment=None, **kwargs): """ - Initialize the model from an iterable of `sentences`. Each sentence is a - LabeledSentence object that will be used for training. + Initialize the model from an iterable of `documents`. Each document is a + TaggedDocument object that will be used for training. - The `sentences` iterable can be simply a list of LabeledSentence elements, but for larger corpora, - consider an iterable that streams the sentences directly from disk/network. + The `documents` iterable can be simply a list of TaggedDocument elements, but for larger corpora, + consider an iterable that streams the documents directly from disk/network. - If you don't supply `sentences`, the model is left uninitialized -- use if + If you don't supply `documents`, the model is left uninitialized -- use if you plan to initialize it in some other way. - `dm` defines the training algorithm. By default (`dm=1`), distributed memory is used. - Otherwise, `dbow` is employed. + `dm` defines the training algorithm. By default (`dm=1`), 'distributed memory' (PV-DM) is used. + Otherwise, `distributed bag of words` (PV-DBOW) is employed. `size` is the dimensionality of the feature vectors. - `window` is the maximum distance between the current and predicted word within a sentence. + `window` is the maximum distance between the predicted word and context words used for prediction + within a document. `alpha` is the initial learning rate (will linearly drop to zero as training progresses). - `seed` = for the random number generator. + `seed` = for the random number generator. Only runs with a single worker will be + deterministically reproducible because of the ordering randomness in multi-threaded runs. `min_count` = ignore all words with total frequency lower than this. @@ -181,68 +496,175 @@ def __init__(self, sentences=None, size=300, alpha=0.025, window=8, min_count=5, specifies how many "noise words" should be drawn (usually between 5-20). `dm_mean` = if 0 (default), use the sum of the context word vectors. If 1, use the mean. - Only applies when dm is used. + Only applies when dm is used in non-concatenative mode. + + `dm_concat` = if 1, use concatenation of context vectors rather than sum/average; + default is 0 (off). Note concatenation results in a much-larger model, as the input + is no longer the size of one (sampled or arithmatically combined) word vector, but the + size of the tag(s) and all words in the context strung together. + + `dm_tag_count` = expected constant number of document tags per document, when using + dm_concat mode; default is 1. + + `dbow_words` if set to 1 trains word-vectors (in skip-gram fashion) simultaneous with DBOW + doc-vector training; default is 0 (faster training of doc-vectors only). """ Word2Vec.__init__(self, size=size, alpha=alpha, window=window, min_count=min_count, sample=sample, seed=seed, workers=workers, min_alpha=min_alpha, - sg=(1+dm) % 2, hs=hs, negative=negative, cbow_mean=dm_mean, **kwargs) - self.train_words = train_words - self.train_lbls = train_lbls - if sentences is not None: - self.build_vocab(sentences) - self.train(sentences) - - @staticmethod - def _vocab_from(sentences): - sentence_no, vocab = -1, {} + sg=(1+dm) % 2, hs=hs, negative=negative, cbow_mean=dm_mean, + null_word=dm_concat, **kwargs) + self.dbow_words = dbow_words + self.dm_concat = dm_concat + self.dm_tag_count = dm_tag_count + self.docvecs = docvecs + if not self.docvecs: + self.docvecs = DocvecsArray(docvecs_mapfile) + self.comment = comment + if documents is not None: + self.build_vocab(documents) + self.train(documents) + + def clear_sims(self): + Word2Vec.clear_sims(self) + self.docvecs.clear_sims() + + def reset_weights(self): + if self.dm_concat: + # expand l1 size to match concatenated tags+words length + self.layer1_size = (self.dm_tag_count + (2 * self.window)) * self.vector_size + logger.info("using concatenative %d-dimensional layer1"% (self.layer1_size)) + Word2Vec.reset_weights(self) + self.docvecs.reset_weights(self) + + def reset_from(self, other_model): + """Reuse shareable structures from other_model.""" + self.docvecs.borrow_from(other_model.docvecs) + Word2Vec.reset_from(self, other_model) + + def _vocab_from(self, documents): + document_no, vocab = -1, {} total_words = 0 - for sentence_no, sentence in enumerate(sentences): - if sentence_no % 10000 == 0: - logger.info("PROGRESS: at item #%i, processed %i words and %i word types" % - (sentence_no, total_words, len(vocab))) - sentence_length = len(sentence.words) - for label in sentence.labels: - total_words += 1 - if label in vocab: - vocab[label].count += sentence_length - else: - vocab[label] = Vocab(count=sentence_length) - for word in sentence.words: + for document_no, document in enumerate(documents): + if document_no % 10000 == 0: + logger.info("PROGRESS: at document #%i, processed %i words and %i word types" % + (document_no, total_words, len(vocab))) + document_length = len(document.words) + for tag in document.tags: + self.docvecs.note_doctag(tag, document_no, document_length) + for word in document.words: total_words += 1 if word in vocab: vocab[word].count += 1 else: vocab[word] = Vocab(count=1) - logger.info("collected %i word types from a corpus of %i words and %i items" % - (len(vocab), total_words, sentence_no + 1)) + logger.info("collected %i word types from a corpus of %i words and %i documents" % + (len(vocab), total_words, document_no + 1)) return vocab - def _prepare_sentences(self, sentences): - for sentence in sentences: - # avoid calling random_sample() where prob >= 1, to speed things up a little: - sampled = [self.vocab[word] for word in sentence.words - if word in self.vocab and (self.vocab[word].sample_probability >= 1.0 or - self.vocab[word].sample_probability >= random.random_sample())] - yield (sampled, [self.vocab[word] for word in sentence.labels if word in self.vocab]) + def _prepare_items(self, documents): + for document in documents: + yield (self._tokens_to_vocabs(document.words), + self.docvecs.indexed_doctags(document.tags)) + + def _tokens_to_vocabs(self, tokens, sample=True, source_dict=None): + """Convert list of tokens to items (Vocabs) from source_dict.""" + if source_dict is None: + source_dict = self.vocab + if sample: + return [source_dict[token] for token in tokens if token in source_dict + and (source_dict[token].sample_probability >= 1.0 or + source_dict[token].sample_probability >= random.random_sample())] + else: + return [source_dict[token] for token in tokens if token in source_dict] def _get_job_words(self, alpha, work, job, neu1): if self.sg: - return sum(train_sentence_dbow(self, sentence, lbls, alpha, work, self.train_words, self.train_lbls) for sentence, lbls in job) + tally = sum(train_document_dbow(self, word_vocabs, doctag_indexes, alpha, work, train_words=self.dbow_words, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + for word_vocabs, (doctag_indexes, doctag_vectors, doctag_locks, ignored) in job) + elif self.dm_concat: + tally = sum(train_document_dm_concat(self, word_vocabs, doctag_indexes, alpha, work, neu1, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + for word_vocabs, (doctag_indexes, doctag_vectors, doctag_locks, ignored) in job) else: - return sum(train_sentence_dm(self, sentence, lbls, alpha, work, neu1, self.train_words, self.train_lbls) for sentence, lbls in job) - - def __str__(self): - return "Doc2Vec(vocab=%s, size=%s, alpha=%s)" % (len(self.index2word), self.layer1_size, self.alpha) + tally = sum(train_document_dm(self, word_vocabs, doctag_indexes, alpha, work, neu1, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + for word_vocabs, (doctag_indexes, doctag_vectors, doctag_locks, ignored) in job) + self.docvecs.trained_items(item for s, item in job) + return tally - def save(self, *args, **kwargs): - kwargs['ignore'] = kwargs.get('ignore', ['syn0norm']) # don't bother storing the cached normalized vectors - super(Doc2Vec, self).save(*args, **kwargs) + def infer_vector(self, document, alpha=0.1, min_alpha=0.0001, steps=5): + """ + Infer a vector for given post-bulk training document. + Document should be a list of (word) tokens. + """ + doctag_vectors = empty((1, self.vector_size), dtype=REAL) + doctag_vectors[0] = self.seeded_vector(' '.join(document)) + doctag_locks = ones(1, dtype=REAL) + doctag_indexes = [0] + word_vocabs = self._tokens_to_vocabs(document) + + work = zeros(self.layer1_size, dtype=REAL) + if not self.sg: + neu1 = matutils.zeros_aligned(self.layer1_size, dtype=REAL) + + for i in range(steps): + if self.sg: + train_document_dbow(self, word_vocabs, doctag_indexes, alpha, work, + learn_words=False, learn_hidden=False, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + elif self.dm_concat: + train_document_dm_concat(self, word_vocabs, doctag_indexes, alpha, work, neu1, + learn_words=False, learn_hidden=False, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + else: + train_document_dm(self, word_vocabs, doctag_indexes, alpha, work, neu1, + learn_words=False, learn_hidden=False, + doctag_vectors=doctag_vectors, doctag_locks=doctag_locks) + alpha = ((alpha - min_alpha) / (steps - i)) + min_alpha + + return doctag_vectors[0] -class LabeledBrownCorpus(object): - """Iterate over sentences from the Brown corpus (part of NLTK data), yielding - each sentence out as a LabeledSentence object.""" + def __str__(self): + """Abbreviated name reflecting major configuration paramaters.""" + segments = [] + if self.comment: + segments.append('"%s"' % self.comment) + if self.sg: + if self.dbow_words: + segments.append('dbow+w') # also training words + else: + segments.append('dbow') # PV-DBOW (skip-gram-style) + + else: # PV-DM... + if self.dm_concat: + segments.append('dm/c') # ...with concatenative context layer + else: + if self.cbow_mean: + segments.append('dm/m') + else: + segments.append('dm/s') + segments.append('d%d' % self.vector_size) # dimensions + if self.negative: + segments.append('n%d' % self.negative) # negative samples + if self.hs: + segments.append('hs') + if not self.sg or (self.sg and self.dbow_words): + segments.append('w%d' % self.window) # window size, when relevant + if self.min_count > 1: + segments.append('mc%d' % self.min_count) + if self.sample > 0: + segments.append('s%E' % self.sample) + if self.workers > 1: + segments.append('t%d' % self.workers) + return 'Doc2Vec(%s)' % ','.join(segments) + + +class TaggedBrownCorpus(object): + """Iterate over documents from the Brown corpus (part of NLTK data), yielding + each document out as a TaggedDocument object.""" def __init__(self, dirname): self.dirname = dirname @@ -253,33 +675,33 @@ def __iter__(self): continue for item_no, line in enumerate(utils.smart_open(fname)): line = utils.to_unicode(line) - # each file line is a single sentence in the Brown corpus + # each file line is a single document in the Brown corpus # each token is WORD/POS_TAG token_tags = [t.split('/') for t in line.split() if len(t.split('/')) == 2] # ignore words with non-alphabetic tags like ",", "!" etc (punctuation, weird stuff) words = ["%s/%s" % (token.lower(), tag[:2]) for token, tag in token_tags if tag[:2].isalpha()] - if not words: # don't bother sending out empty sentences + if not words: # don't bother sending out empty documents continue - yield LabeledSentence(words, ['%s_SENT_%s' % (fname, item_no)]) + yield TaggedDocument(words, ['%s_SENT_%s' % (fname, item_no)]) -class LabeledLineSentence(object): - """Simple format: one sentence = one line = one LabeledSentence object. +class TaggedLineDocument(object): + """Simple format: one document = one line = one TaggedDocument object. Words are expected to be already preprocessed and separated by whitespace, - labels are constructed automatically from the sentence line number.""" + tags are constructed automatically from the document line number.""" def __init__(self, source): """ `source` can be either a string (filename) or a file object. Example:: - sentences = LineSentence('myfile.txt') + documents = TaggedLineDocument('myfile.txt') Or for compressed files:: - sentences = LineSentence('compressed_text.txt.bz2') - sentences = LineSentence('compressed_text.txt.gz') + documents = TaggedLineDocument('compressed_text.txt.bz2') + documents = TaggedLineDocument('compressed_text.txt.gz') """ self.source = source @@ -291,9 +713,9 @@ def __iter__(self): # Things that don't have seek will trigger an exception self.source.seek(0) for item_no, line in enumerate(self.source): - yield LabeledSentence(utils.to_unicode(line).split(), ['SENT_%s' % item_no]) + yield TaggedDocument(utils.to_unicode(line).split(), [item_no]) except AttributeError: # If it didn't work like a file, use it as a string filename with utils.smart_open(self.source) as fin: for item_no, line in enumerate(fin): - yield LabeledSentence(utils.to_unicode(line).split(), ['SENT_%s' % item_no]) + yield TaggedDocument(utils.to_unicode(line).split(), [item_no]) diff --git a/gensim/models/doc2vec_inner.c b/gensim/models/doc2vec_inner.c index 48bdfbaf75..65a888fc56 100644 --- a/gensim/models/doc2vec_inner.c +++ b/gensim/models/doc2vec_inner.c @@ -648,12 +648,12 @@ typedef npy_double __pyx_t_5numpy_double_t; */ typedef npy_longdouble __pyx_t_5numpy_longdouble_t; -/* "trunk/gensim/models/doc2vec_inner.pyx":23 +/* "trunk/gensim/models/doc2vec_inner.pyx":24 * * REAL = np.float32 * ctypedef np.float32_t REAL_t # <<<<<<<<<<<<<< * - * DEF MAX_SENTENCE_LEN = 10000 + * DEF MAX_DOCUMENT_LEN = 10000 */ typedef __pyx_t_5numpy_float32_t __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t; #if CYTHON_CCOMPLEX @@ -715,8 +715,8 @@ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; */ typedef npy_cdouble __pyx_t_5numpy_complex_t; -/* "trunk/gensim/models/doc2vec_inner.pyx":27 - * DEF MAX_SENTENCE_LEN = 10000 +/* "trunk/gensim/models/doc2vec_inner.pyx":28 + * DEF MAX_DOCUMENT_LEN = 10000 * * ctypedef void (*scopy_ptr) (const int *N, const float *X, const int *incX, float *Y, const int *incY) nogil # <<<<<<<<<<<<<< * ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil @@ -724,7 +724,7 @@ typedef npy_cdouble __pyx_t_5numpy_complex_t; */ typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_scopy_ptr)(int const *, float const *, int const *, float *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":28 +/* "trunk/gensim/models/doc2vec_inner.pyx":29 * * ctypedef void (*scopy_ptr) (const int *N, const float *X, const int *incX, float *Y, const int *incY) nogil * ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil # <<<<<<<<<<<<<< @@ -733,7 +733,7 @@ typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_scopy_ptr)(int con */ typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_saxpy_ptr)(int const *, float const *, float const *, int const *, float *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":29 +/* "trunk/gensim/models/doc2vec_inner.pyx":30 * ctypedef void (*scopy_ptr) (const int *N, const float *X, const int *incX, float *Y, const int *incY) nogil * ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil * ctypedef float (*sdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil # <<<<<<<<<<<<<< @@ -742,7 +742,7 @@ typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_saxpy_ptr)(int con */ typedef float (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sdot_ptr)(int const *, float const *, int const *, float const *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":30 +/* "trunk/gensim/models/doc2vec_inner.pyx":31 * ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil * ctypedef float (*sdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil * ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil # <<<<<<<<<<<<<< @@ -751,7 +751,7 @@ typedef float (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sdot_ptr)(int con */ typedef double (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_dsdot_ptr)(int const *, float const *, int const *, float const *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":31 +/* "trunk/gensim/models/doc2vec_inner.pyx":32 * ctypedef float (*sdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil * ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil * ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil # <<<<<<<<<<<<<< @@ -760,50 +760,32 @@ typedef double (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_dsdot_ptr)(int c */ typedef double (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_snrm2_ptr)(int const *, float const *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":32 +/* "trunk/gensim/models/doc2vec_inner.pyx":33 * ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil * ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil # <<<<<<<<<<<<<< * - * ctypedef void (*fast_sentence_dbow_hs_ptr) ( + * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x */ typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sscal_ptr)(int const *, float const *, float const *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":34 - * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil - * - * ctypedef void (*fast_sentence_dbow_hs_ptr) ( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, - */ -typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs_ptr)(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int); - -/* "trunk/gensim/models/doc2vec_inner.pyx":39 - * const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, int tw, int tl) nogil +/* "trunk/gensim/models/doc2vec_inner.pyx":51 * - * ctypedef unsigned long long (*fast_sentence_dbow_neg_ptr) ( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ -typedef unsigned PY_LONG_LONG (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg_ptr)(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, unsigned PY_LONG_LONG, int, int); - -/* "trunk/gensim/models/doc2vec_inner.pyx":45 - * unsigned long long next_random, int tw, int tl) nogil + * # function implementations swapped based on BLAS detected + * ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil # <<<<<<<<<<<<<< + * ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil * - * ctypedef void (*fast_sentence_dm_hs_ptr) ( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, */ -typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs_ptr)(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t *, __pyx_t_5numpy_uint32_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, int, int, int); +typedef __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_our_dot_ptr)(int const *, float const *, int const *, float const *, int const *); -/* "trunk/gensim/models/doc2vec_inner.pyx":51 - * REAL_t *work, int i, int j, int k, int cbow_mean, int lbl_length, int tw, int tl) nogil +/* "trunk/gensim/models/doc2vec_inner.pyx":52 + * # function implementations swapped based on BLAS detected + * ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil + * ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil # <<<<<<<<<<<<<< * - * ctypedef unsigned long long (*fast_sentence_dm_neg_ptr) ( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * cdef our_dot_ptr our_dot */ -typedef unsigned PY_LONG_LONG (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg_ptr)(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t *, __pyx_t_5numpy_uint32_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG, int, int, int); +typedef void (*__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy_ptr)(int const *, float const *, float const *, int const *, float *, int const *); /* --- Runtime support code (head) --- */ #ifndef CYTHON_REFNANNY @@ -1160,25 +1142,21 @@ static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sdot_ptr __pyx_v_5trunk_6g static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_dsdot_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot; static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_snrm2_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_snrm2; static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sscal_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal; -static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs; -static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg; -static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs; -static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg; static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[1000]; static int __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE; static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, unsigned PY_LONG_LONG, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, unsigned PY_LONG_LONG, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, unsigned PY_LONG_LONG, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, int, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, int, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, int, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t *, __pyx_t_5numpy_uint32_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG, int, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t *, __pyx_t_5numpy_uint32_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG, int, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, int *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t *, __pyx_t_5numpy_uint32_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG, int, int, int); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_our_dot_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot; +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy_ptr __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy; +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_double(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_float(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_noblas(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy_noblas(int const *, float const *, float const *, int const *, float *, int const *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int, int, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *); /*proto*/ +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, unsigned PY_LONG_LONG, int, int, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , int); /*proto*/ +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , int); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , int const , int); /*proto*/ +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *, int const , int const , int); /*proto*/ #define __Pyx_MODULE_NAME "trunk.gensim.models.doc2vec_inner" int __pyx_module_is_main_trunk__gensim__models__doc2vec_inner = 0; @@ -1187,9 +1165,10 @@ static PyObject *__pyx_builtin_range; static PyObject *__pyx_builtin_enumerate; static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_RuntimeError; -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_dbow(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_sentence, PyObject *__pyx_v_lbls, PyObject *__pyx_v_alpha, PyObject *__pyx_v__work, PyObject *__pyx_v_train_words, PyObject *__pyx_v_train_lbls); /* proto */ -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence_dm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_sentence, PyObject *__pyx_v_lbls, PyObject *__pyx_v_alpha, PyObject *__pyx_v__work, PyObject *__pyx_v__neu1, PyObject *__pyx_v_train_words, PyObject *__pyx_v_train_lbls); /* proto */ -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_document_dbow(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_train_words, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks); /* proto */ +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_document_dm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_neu1, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks); /* proto */ +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4train_document_dm_concat(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_neu1, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks); /* proto */ +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_6init(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static char __pyx_k_B[] = "B"; @@ -1207,33 +1186,35 @@ static char __pyx_k_i[] = "i"; static char __pyx_k_j[] = "j"; static char __pyx_k_k[] = "k"; static char __pyx_k_l[] = "l"; +static char __pyx_k_m[] = "m"; +static char __pyx_k_n[] = "n"; static char __pyx_k_q[] = "q"; static char __pyx_k_x[] = "x"; static char __pyx_k_y[] = "y"; static char __pyx_k_Zd[] = "Zd"; static char __pyx_k_Zf[] = "Zf"; static char __pyx_k_Zg[] = "Zg"; +static char __pyx_k__5[] = "\000"; static char __pyx_k_hs[] = "hs"; static char __pyx_k_np[] = "np"; -static char __pyx_k_tl[] = "tl"; -static char __pyx_k_tw[] = "tw"; static char __pyx_k_REAL[] = "REAL"; static char __pyx_k_code[] = "code"; static char __pyx_k_init[] = "init"; static char __pyx_k_item[] = "item"; -static char __pyx_k_lbls[] = "lbls"; static char __pyx_k_main[] = "__main__"; -static char __pyx_k_neu1[] = "_neu1"; +static char __pyx_k_neu1[] = "neu1"; static char __pyx_k_sdot[] = "sdot"; static char __pyx_k_size[] = "size"; static char __pyx_k_syn0[] = "syn0"; static char __pyx_k_syn1[] = "syn1"; static char __pyx_k_test[] = "__test__"; static char __pyx_k_word[] = "word"; -static char __pyx_k_work[] = "_work"; +static char __pyx_k_work[] = "work"; static char __pyx_k_alpha[] = "alpha"; static char __pyx_k_codes[] = "codes"; +static char __pyx_k_count[] = "count"; static char __pyx_k_d_res[] = "d_res"; +static char __pyx_k_dtype[] = "dtype"; static char __pyx_k_fblas[] = "fblas"; static char __pyx_k_index[] = "index"; static char __pyx_k_model[] = "model"; @@ -1246,14 +1227,17 @@ static char __pyx_k_scopy[] = "scopy"; static char __pyx_k_snrm2[] = "snrm2"; static char __pyx_k_sscal[] = "sscal"; static char __pyx_k_table[] = "table"; +static char __pyx_k_vocab[] = "vocab"; +static char __pyx_k_zeros[] = "zeros"; static char __pyx_k_import[] = "__import__"; -static char __pyx_k_neu1_2[] = "neu1"; +static char __pyx_k_neu1_2[] = "_neu1"; static char __pyx_k_points[] = "points"; static char __pyx_k_random[] = "random"; static char __pyx_k_result[] = "result"; static char __pyx_k_window[] = "window"; -static char __pyx_k_work_2[] = "work"; +static char __pyx_k_work_2[] = "_work"; static char __pyx_k_alpha_2[] = "_alpha"; +static char __pyx_k_docvecs[] = "docvecs"; static char __pyx_k_float32[] = "float32"; static char __pyx_k_indexes[] = "indexes"; static char __pyx_k_randint[] = "randint"; @@ -1262,27 +1246,50 @@ static char __pyx_k_codelens[] = "codelens"; static char __pyx_k_cpointer[] = "_cpointer"; static char __pyx_k_expected[] = "expected"; static char __pyx_k_negative[] = "negative"; -static char __pyx_k_sentence[] = "sentence"; static char __pyx_k_cbow_mean[] = "cbow_mean"; static char __pyx_k_enumerate[] = "enumerate"; -static char __pyx_k_lbl_codes[] = "lbl_codes"; +static char __pyx_k_inv_count[] = "inv_count"; static char __pyx_k_table_len[] = "table_len"; static char __pyx_k_ValueError[] = "ValueError"; -static char __pyx_k_lbl_length[] = "lbl_length"; -static char __pyx_k_lbl_points[] = "lbl_points"; -static char __pyx_k_train_lbls[] = "train_lbls"; +static char __pyx_k_doctag_len[] = "doctag_len"; +static char __pyx_k_syn0_lockf[] = "syn0_lockf"; +static char __pyx_k_word_locks[] = "word_locks"; +static char __pyx_k_doctag_syn0[] = "doctag_syn0"; static char __pyx_k_layer1_size[] = "layer1_size"; -static char __pyx_k_lbl_indexes[] = "lbl_indexes"; +static char __pyx_k_learn_words[] = "learn_words"; static char __pyx_k_next_random[] = "next_random"; static char __pyx_k_train_words[] = "train_words"; +static char __pyx_k_vector_size[] = "vector_size"; +static char __pyx_k_word_vocabs[] = "word_vocabs"; static char __pyx_k_FAST_VERSION[] = "FAST_VERSION"; static char __pyx_k_RuntimeError[] = "RuntimeError"; -static char __pyx_k_lbl_codelens[] = "lbl_codelens"; -static char __pyx_k_sentence_len[] = "sentence_len"; +static char __pyx_k_dm_tag_count[] = "dm_tag_count"; +static char __pyx_k_doctag_locks[] = "doctag_locks"; +static char __pyx_k_document_len[] = "document_len"; +static char __pyx_k_learn_hidden[] = "learn_hidden"; +static char __pyx_k_predict_word[] = "predict_word"; +static char __pyx_k_word_locks_2[] = "_word_locks"; +static char __pyx_k_word_vectors[] = "word_vectors"; +static char __pyx_k_learn_doctags[] = "learn_doctags"; +static char __pyx_k_learn_words_2[] = "_learn_words"; +static char __pyx_k_train_words_2[] = "_train_words"; +static char __pyx_k_doctag_indexes[] = "doctag_indexes"; +static char __pyx_k_doctag_locks_2[] = "_doctag_locks"; +static char __pyx_k_doctag_vectors[] = "doctag_vectors"; +static char __pyx_k_learn_hidden_2[] = "_learn_hidden"; +static char __pyx_k_window_indexes[] = "window_indexes"; +static char __pyx_k_word_vectors_2[] = "_word_vectors"; +static char __pyx_k_learn_doctags_2[] = "_learn_doctags"; +static char __pyx_k_null_word_index[] = "null_word_index"; static char __pyx_k_reduced_windows[] = "reduced_windows"; +static char __pyx_k_doctag_indexes_2[] = "_doctag_indexes"; +static char __pyx_k_doctag_vectors_2[] = "_doctag_vectors"; +static char __pyx_k_doctag_syn0_lockf[] = "doctag_syn0_lockf"; static char __pyx_k_scipy_linalg_blas[] = "scipy.linalg.blas"; -static char __pyx_k_train_sentence_dm[] = "train_sentence_dm"; -static char __pyx_k_train_sentence_dbow[] = "train_sentence_dbow"; +static char __pyx_k_train_document_dm[] = "train_document_dm"; +static char __pyx_k_expected_doctag_len[] = "expected_doctag_len"; +static char __pyx_k_train_document_dbow[] = "train_document_dbow"; +static char __pyx_k_train_document_dm_concat[] = "train_document_dm_concat"; static char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; static char __pyx_k_Users_scratch_Documents_dev2015[] = "/Users/scratch/Documents/dev2015/gensim_venv/src/trunk/gensim/models/doc2vec_inner.pyx"; static char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; @@ -1299,16 +1306,32 @@ static PyObject *__pyx_n_s_REAL; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_kp_s_Users_scratch_Documents_dev2015; static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_kp_s__5; static PyObject *__pyx_n_s_alpha; static PyObject *__pyx_n_s_alpha_2; static PyObject *__pyx_n_s_cbow_mean; static PyObject *__pyx_n_s_code; static PyObject *__pyx_n_s_codelens; static PyObject *__pyx_n_s_codes; +static PyObject *__pyx_n_s_count; static PyObject *__pyx_n_s_cpointer; static PyObject *__pyx_n_s_d_res; +static PyObject *__pyx_n_s_dm_tag_count; +static PyObject *__pyx_n_s_doctag_indexes; +static PyObject *__pyx_n_s_doctag_indexes_2; +static PyObject *__pyx_n_s_doctag_len; +static PyObject *__pyx_n_s_doctag_locks; +static PyObject *__pyx_n_s_doctag_locks_2; +static PyObject *__pyx_n_s_doctag_syn0; +static PyObject *__pyx_n_s_doctag_syn0_lockf; +static PyObject *__pyx_n_s_doctag_vectors; +static PyObject *__pyx_n_s_doctag_vectors_2; +static PyObject *__pyx_n_s_document_len; +static PyObject *__pyx_n_s_docvecs; +static PyObject *__pyx_n_s_dtype; static PyObject *__pyx_n_s_enumerate; static PyObject *__pyx_n_s_expected; +static PyObject *__pyx_n_s_expected_doctag_len; static PyObject *__pyx_n_s_fblas; static PyObject *__pyx_n_s_float32; static PyObject *__pyx_n_s_hs; @@ -1317,18 +1340,21 @@ static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_index; static PyObject *__pyx_n_s_indexes; static PyObject *__pyx_n_s_init; +static PyObject *__pyx_n_s_inv_count; static PyObject *__pyx_n_s_item; static PyObject *__pyx_n_s_j; static PyObject *__pyx_n_s_k; static PyObject *__pyx_n_s_layer1_size; -static PyObject *__pyx_n_s_lbl_codelens; -static PyObject *__pyx_n_s_lbl_codes; -static PyObject *__pyx_n_s_lbl_indexes; -static PyObject *__pyx_n_s_lbl_length; -static PyObject *__pyx_n_s_lbl_points; -static PyObject *__pyx_n_s_lbls; +static PyObject *__pyx_n_s_learn_doctags; +static PyObject *__pyx_n_s_learn_doctags_2; +static PyObject *__pyx_n_s_learn_hidden; +static PyObject *__pyx_n_s_learn_hidden_2; +static PyObject *__pyx_n_s_learn_words; +static PyObject *__pyx_n_s_learn_words_2; +static PyObject *__pyx_n_s_m; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_model; +static PyObject *__pyx_n_s_n; static PyObject *__pyx_kp_u_ndarray_is_not_C_contiguous; static PyObject *__pyx_kp_u_ndarray_is_not_Fortran_contiguou; static PyObject *__pyx_n_s_negative; @@ -1336,10 +1362,12 @@ static PyObject *__pyx_n_s_neu1; static PyObject *__pyx_n_s_neu1_2; static PyObject *__pyx_n_s_next_random; static PyObject *__pyx_n_s_np; +static PyObject *__pyx_n_s_null_word_index; static PyObject *__pyx_n_s_numpy; static PyObject *__pyx_n_s_p_res; static PyObject *__pyx_n_s_point; static PyObject *__pyx_n_s_points; +static PyObject *__pyx_n_s_predict_word; static PyObject *__pyx_n_s_randint; static PyObject *__pyx_n_s_random; static PyObject *__pyx_n_s_range; @@ -1349,31 +1377,38 @@ static PyObject *__pyx_n_s_saxpy; static PyObject *__pyx_n_s_scipy_linalg_blas; static PyObject *__pyx_n_s_scopy; static PyObject *__pyx_n_s_sdot; -static PyObject *__pyx_n_s_sentence; -static PyObject *__pyx_n_s_sentence_len; static PyObject *__pyx_n_s_size; static PyObject *__pyx_n_s_snrm2; static PyObject *__pyx_n_s_sscal; static PyObject *__pyx_n_s_syn0; +static PyObject *__pyx_n_s_syn0_lockf; static PyObject *__pyx_n_s_syn1; static PyObject *__pyx_n_s_syn1neg; static PyObject *__pyx_n_s_table; static PyObject *__pyx_n_s_table_len; static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_tl; -static PyObject *__pyx_n_s_train_lbls; -static PyObject *__pyx_n_s_train_sentence_dbow; -static PyObject *__pyx_n_s_train_sentence_dm; +static PyObject *__pyx_n_s_train_document_dbow; +static PyObject *__pyx_n_s_train_document_dm; +static PyObject *__pyx_n_s_train_document_dm_concat; static PyObject *__pyx_n_s_train_words; +static PyObject *__pyx_n_s_train_words_2; static PyObject *__pyx_n_s_trunk_gensim_models_doc2vec_inne; -static PyObject *__pyx_n_s_tw; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; +static PyObject *__pyx_n_s_vector_size; +static PyObject *__pyx_n_s_vocab; static PyObject *__pyx_n_s_window; +static PyObject *__pyx_n_s_window_indexes; static PyObject *__pyx_n_s_word; +static PyObject *__pyx_n_s_word_locks; +static PyObject *__pyx_n_s_word_locks_2; +static PyObject *__pyx_n_s_word_vectors; +static PyObject *__pyx_n_s_word_vectors_2; +static PyObject *__pyx_n_s_word_vocabs; static PyObject *__pyx_n_s_work; static PyObject *__pyx_n_s_work_2; static PyObject *__pyx_n_s_x; static PyObject *__pyx_n_s_y; +static PyObject *__pyx_n_s_zeros; static PyObject *__pyx_int_0; static PyObject *__pyx_int_1; static PyObject *__pyx_int_2; @@ -1382,206 +1417,209 @@ static PyObject *__pyx_tuple_; static PyObject *__pyx_tuple__2; static PyObject *__pyx_tuple__3; static PyObject *__pyx_tuple__4; -static PyObject *__pyx_tuple__5; static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; static PyObject *__pyx_tuple__10; static PyObject *__pyx_tuple__11; +static PyObject *__pyx_tuple__12; static PyObject *__pyx_tuple__13; -static PyObject *__pyx_tuple__15; -static PyObject *__pyx_codeobj__12; -static PyObject *__pyx_codeobj__14; -static PyObject *__pyx_codeobj__16; +static PyObject *__pyx_tuple__14; +static PyObject *__pyx_tuple__16; +static PyObject *__pyx_tuple__18; +static PyObject *__pyx_tuple__20; +static PyObject *__pyx_codeobj__15; +static PyObject *__pyx_codeobj__17; +static PyObject *__pyx_codeobj__19; +static PyObject *__pyx_codeobj__21; -/* "trunk/gensim/models/doc2vec_inner.pyx":76 - * cdef REAL_t ONEF = 1.0 +/* "trunk/gensim/models/doc2vec_inner.pyx":58 + * + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return dsdot(N, X, incX, Y, incY) * - * cdef void fast_sentence0_dbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, */ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row1; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_double(int const *__pyx_v_N, float const *__pyx_v_X, int const *__pyx_v_incX, float const *__pyx_v_Y, int const *__pyx_v_incY) { + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_r; - /* "trunk/gensim/models/doc2vec_inner.pyx":82 - * - * cdef long long a, b - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef REAL_t f, g + /* "trunk/gensim/models/doc2vec_inner.pyx":59 + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + * return dsdot(N, X, incX, Y, incY) # <<<<<<<<<<<<<< * + * # for when fblas.sdot returns a float */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); + __pyx_r = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot(__pyx_v_N, __pyx_v_X, __pyx_v_incX, __pyx_v_Y, __pyx_v_incY)); + goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":85 - * cdef REAL_t f, g + /* "trunk/gensim/models/doc2vec_inner.pyx":58 + * + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return dsdot(N, X, incX, Y, incY) * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelen): - * row2 = word_point[b] * size */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - /* "trunk/gensim/models/doc2vec_inner.pyx":86 + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/doc2vec_inner.pyx":62 + * + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return sdot(N, X, incX, Y, incY) * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelen): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) */ - __pyx_t_1 = __pyx_v_codelen; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":87 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelen): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_float(int const *__pyx_v_N, float const *__pyx_v_X, int const *__pyx_v_incX, float const *__pyx_v_Y, int const *__pyx_v_incY) { + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_r; + + /* "trunk/gensim/models/doc2vec_inner.pyx":63 + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + * return sdot(N, X, incX, Y, incY) # <<<<<<<<<<<<<< + * + * # for when no blas available */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); + __pyx_r = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot(__pyx_v_N, __pyx_v_X, __pyx_v_incX, __pyx_v_Y, __pyx_v_incY)); + goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":88 - * for b in range(codelen): - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":62 + * + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return sdot(N, X, incX, Y, incY) + * */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); - /* "trunk/gensim/models/doc2vec_inner.pyx":89 - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/doc2vec_inner.pyx":66 + * + * # for when no blas available + * cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * # not a true full dot()-implementation: just enough for our cases + * cdef int i */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":90 - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha +static __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_noblas(int const *__pyx_v_N, float const *__pyx_v_X, CYTHON_UNUSED int const *__pyx_v_incX, float const *__pyx_v_Y, CYTHON_UNUSED int const *__pyx_v_incY) { + int __pyx_v_i; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_a; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_r; + int __pyx_t_1; + + /* "trunk/gensim/models/doc2vec_inner.pyx":70 + * cdef int i + * cdef REAL_t a + * a = 0.0 # <<<<<<<<<<<<<< + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] */ - goto __pyx_L3_continue; - } + __pyx_v_a = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - /* "trunk/gensim/models/doc2vec_inner.pyx":91 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":71 + * cdef REAL_t a + * a = 0.0 + * for i from 0 <= i < N[0] by 1: # <<<<<<<<<<<<<< + * a += X[i] * Y[i] + * return a */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); + __pyx_t_1 = (__pyx_v_N[0]); + for (__pyx_v_i = 0; __pyx_v_i < __pyx_t_1; __pyx_v_i+=1) { - /* "trunk/gensim/models/doc2vec_inner.pyx":92 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: + /* "trunk/gensim/models/doc2vec_inner.pyx":72 + * a = 0.0 + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] # <<<<<<<<<<<<<< + * return a + * */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); + __pyx_v_a = (__pyx_v_a + ((__pyx_v_X[__pyx_v_i]) * (__pyx_v_Y[__pyx_v_i]))); + } - /* "trunk/gensim/models/doc2vec_inner.pyx":93 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":73 + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] + * return a # <<<<<<<<<<<<<< + * + * # for when no blas available */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_r = __pyx_v_a; + goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":94 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: + /* "trunk/gensim/models/doc2vec_inner.pyx":66 + * + * # for when no blas available + * cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * # not a true full dot()-implementation: just enough for our cases + * cdef int i */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":95 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/doc2vec_inner.pyx":76 + * + * # for when no blas available + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * cdef int i + * for i from 0 <= i < N[0] by 1: */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L8; - } - __pyx_L8:; - __pyx_L3_continue:; - } - /* "trunk/gensim/models/doc2vec_inner.pyx":96 - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy_noblas(int const *__pyx_v_N, float const *__pyx_v_alpha, float const *__pyx_v_X, int const *__pyx_v_incX, float *__pyx_v_Y, int const *__pyx_v_incY) { + int __pyx_v_i; + int __pyx_t_1; + + /* "trunk/gensim/models/doc2vec_inner.pyx":78 + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: + * cdef int i + * for i from 0 <= i < N[0] by 1: # <<<<<<<<<<<<<< + * Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] * */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { + __pyx_t_1 = (__pyx_v_N[0]); + for (__pyx_v_i = 0; __pyx_v_i < __pyx_t_1; __pyx_v_i+=1) { - /* "trunk/gensim/models/doc2vec_inner.pyx":97 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":79 + * cdef int i + * for i from 0 <= i < N[0] by 1: + * Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] # <<<<<<<<<<<<<< * * */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L9; + (__pyx_v_Y[(__pyx_v_i * (__pyx_v_incY[0]))]) = (((__pyx_v_alpha[0]) * (__pyx_v_X[(__pyx_v_i * (__pyx_v_incX[0]))])) + (__pyx_v_Y[(__pyx_v_i * (__pyx_v_incY[0]))])); } - __pyx_L9:; /* "trunk/gensim/models/doc2vec_inner.pyx":76 - * cdef REAL_t ONEF = 1.0 * - * cdef void fast_sentence0_dbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * # for when no blas available + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * cdef int i + * for i from 0 <= i < N[0] by 1: */ /* function exit code */ } -/* "trunk/gensim/models/doc2vec_inner.pyx":100 +/* "trunk/gensim/models/doc2vec_inner.pyx":82 * * - * cdef void fast_sentence1_dbow_hs( # <<<<<<<<<<<<<< + * cdef void fast_document_dbow_hs( # <<<<<<<<<<<<<< * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * REAL_t *context_vectors, REAL_t *syn1, const int size, */ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_tw, int __pyx_v_tl) { +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_context_vectors, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_context_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_learn_context, int __pyx_v_learn_hidden, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_context_locks) { PY_LONG_LONG __pyx_v_b; PY_LONG_LONG __pyx_v_row1; PY_LONG_LONG __pyx_v_row2; @@ -1592,16 +1630,16 @@ static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_h int __pyx_t_3; int __pyx_t_4; - /* "trunk/gensim/models/doc2vec_inner.pyx":106 + /* "trunk/gensim/models/doc2vec_inner.pyx":89 * * cdef long long a, b - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< + * cdef long long row1 = context_index * size, row2 # <<<<<<<<<<<<<< * cdef REAL_t f, g * */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); + __pyx_v_row1 = (__pyx_v_context_index * __pyx_v_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":109 + /* "trunk/gensim/models/doc2vec_inner.pyx":92 * cdef REAL_t f, g * * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< @@ -1610,38 +1648,38 @@ static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_h */ memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - /* "trunk/gensim/models/doc2vec_inner.pyx":110 + /* "trunk/gensim/models/doc2vec_inner.pyx":93 * * memset(work, 0, size * cython.sizeof(REAL_t)) * for b in range(codelen): # <<<<<<<<<<<<<< * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) */ __pyx_t_1 = __pyx_v_codelen; for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_b = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":111 + /* "trunk/gensim/models/doc2vec_inner.pyx":94 * memset(work, 0, size * cython.sizeof(REAL_t)) * for b in range(codelen): * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":112 + /* "trunk/gensim/models/doc2vec_inner.pyx":95 * for b in range(codelen): * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_size), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":113 + /* "trunk/gensim/models/doc2vec_inner.pyx":96 * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -1657,8 +1695,8 @@ static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_h __pyx_L6_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":114 - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":97 + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -1667,354 +1705,519 @@ static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_h goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":115 + /* "trunk/gensim/models/doc2vec_inner.pyx":98 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/doc2vec_inner.pyx":116 + /* "trunk/gensim/models/doc2vec_inner.pyx":99 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: */ __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":117 + /* "trunk/gensim/models/doc2vec_inner.pyx":100 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":118 + /* "trunk/gensim/models/doc2vec_inner.pyx":101 * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) + * if learn_context: */ - __pyx_t_3 = (__pyx_v_tw != 0); + __pyx_t_3 = (__pyx_v_learn_hidden != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":119 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":102 + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< + * if learn_context: + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); goto __pyx_L8; } __pyx_L8:; __pyx_L3_continue:; } - /* "trunk/gensim/models/doc2vec_inner.pyx":120 - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":103 + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) + * if learn_context: # <<<<<<<<<<<<<< + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) * */ - __pyx_t_3 = (__pyx_v_tl != 0); + __pyx_t_3 = (__pyx_v_learn_context != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":121 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":104 + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) + * if learn_context: + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) # <<<<<<<<<<<<<< * * */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_context_locks[__pyx_v_context_index])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); goto __pyx_L9; } __pyx_L9:; - /* "trunk/gensim/models/doc2vec_inner.pyx":100 + /* "trunk/gensim/models/doc2vec_inner.pyx":82 * * - * cdef void fast_sentence1_dbow_hs( # <<<<<<<<<<<<<< + * cdef void fast_document_dbow_hs( # <<<<<<<<<<<<<< * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * REAL_t *context_vectors, REAL_t *syn1, const int size, */ /* function exit code */ } -/* "trunk/gensim/models/doc2vec_inner.pyx":124 +/* "trunk/gensim/models/doc2vec_inner.pyx":107 * * - * cdef void fast_sentence2_dbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * cdef unsigned long long fast_document_dbow_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, + * REAL_t *context_vectors, REAL_t *syn1neg, const int size, const np.uint32_t word_index, */ -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_b; +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_context_vectors, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_context_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_learn_context, int __pyx_v_learn_hidden, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_context_locks) { PY_LONG_LONG __pyx_v_row1; PY_LONG_LONG __pyx_v_row2; + unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_label; + __pyx_t_5numpy_uint32_t __pyx_v_target_index; + int __pyx_v_d; + unsigned PY_LONG_LONG __pyx_r; + long __pyx_t_1; + int __pyx_t_2; int __pyx_t_3; - PY_LONG_LONG __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; + int __pyx_t_4; - /* "trunk/gensim/models/doc2vec_inner.pyx":130 - * - * cdef long long a, b - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef REAL_t f, g + /* "trunk/gensim/models/doc2vec_inner.pyx":114 * + * cdef long long a + * cdef long long row1 = context_index * size, row2 # <<<<<<<<<<<<<< + * cdef unsigned long long modulo = 281474976710655ULL + * cdef REAL_t f, g, label */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); + __pyx_v_row1 = (__pyx_v_context_index * __pyx_v_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":133 - * cdef REAL_t f, g - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * for b in range(codelen): + /* "trunk/gensim/models/doc2vec_inner.pyx":115 + * cdef long long a + * cdef long long row1 = context_index * size, row2 + * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< + * cdef REAL_t f, g, label + * cdef np.uint32_t target_index */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_modulo = 281474976710655ULL; - /* "trunk/gensim/models/doc2vec_inner.pyx":134 + /* "trunk/gensim/models/doc2vec_inner.pyx":120 + * cdef int d * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * for b in range(codelen): - * row2 = word_point[b] * size + * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< + * + * for d in range(negative+1): */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } + memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - /* "trunk/gensim/models/doc2vec_inner.pyx":135 - * for a in range(size): - * work[a] = 0.0 - * for b in range(codelen): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = 0.0 + /* "trunk/gensim/models/doc2vec_inner.pyx":122 + * memset(work, 0, size * cython.sizeof(REAL_t)) + * + * for d in range(negative+1): # <<<<<<<<<<<<<< + * if d == 0: + * target_index = word_index */ - __pyx_t_1 = __pyx_v_codelen; + __pyx_t_1 = (__pyx_v_negative + 1); for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":136 - * work[a] = 0.0 - * for b in range(codelen): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); + __pyx_v_d = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":137 - * for b in range(codelen): - * row2 = word_point[b] * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":123 + * + * for d in range(negative+1): + * if d == 0: # <<<<<<<<<<<<<< + * target_index = word_index + * label = ONEF */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); + __pyx_t_3 = ((__pyx_v_d == 0) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":138 - * row2 = word_point[b] * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: + /* "trunk/gensim/models/doc2vec_inner.pyx":124 + * for d in range(negative+1): + * if d == 0: + * target_index = word_index # <<<<<<<<<<<<<< + * label = ONEF + * else: */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; + __pyx_v_target_index = __pyx_v_word_index; - /* "trunk/gensim/models/doc2vec_inner.pyx":139 - * f = 0.0 - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":125 + * if d == 0: + * target_index = word_index + * label = ONEF # <<<<<<<<<<<<<< + * else: + * target_index = table[(next_random >> 16) % table_len] */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]) * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); + __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; + goto __pyx_L5; } + /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":140 - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + /* "trunk/gensim/models/doc2vec_inner.pyx":127 + * label = ONEF + * else: + * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< + * next_random = (next_random * 25214903917ULL + 11) & modulo + * if target_index == word_index: */ - __pyx_t_6 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_6) { - } else { - __pyx_t_5 = __pyx_t_6; - goto __pyx_L10_bool_binop_done; - } - __pyx_t_6 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_5 = __pyx_t_6; - __pyx_L10_bool_binop_done:; - if (__pyx_t_5) { + __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - /* "trunk/gensim/models/doc2vec_inner.pyx":141 - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha + /* "trunk/gensim/models/doc2vec_inner.pyx":128 + * else: + * target_index = table[(next_random >> 16) % table_len] + * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< + * if target_index == word_index: + * continue + */ + __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); + + /* "trunk/gensim/models/doc2vec_inner.pyx":129 + * target_index = table[(next_random >> 16) % table_len] + * next_random = (next_random * 25214903917ULL + 11) & modulo + * if target_index == word_index: # <<<<<<<<<<<<<< + * continue + * label = 0.0 + */ + __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); + if (__pyx_t_3) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":130 + * next_random = (next_random * 25214903917ULL + 11) & modulo + * if target_index == word_index: + * continue # <<<<<<<<<<<<<< + * label = 0.0 + * row2 = target_index * size + */ + goto __pyx_L3_continue; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":131 + * if target_index == word_index: + * continue + * label = 0.0 # <<<<<<<<<<<<<< + * row2 = target_index * size + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + */ + __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); + } + __pyx_L5:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":132 + * continue + * label = 0.0 + * row2 = target_index * size # <<<<<<<<<<<<<< + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: + */ + __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); + + /* "trunk/gensim/models/doc2vec_inner.pyx":133 + * label = 0.0 + * row2 = target_index * size + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * if f <= -MAX_EXP or f >= MAX_EXP: + * continue + */ + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_size), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + + /* "trunk/gensim/models/doc2vec_inner.pyx":134 + * row2 = target_index * size + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< + * continue + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] */ - goto __pyx_L5_continue; + __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); + if (!__pyx_t_4) { + } else { + __pyx_t_3 = __pyx_t_4; + goto __pyx_L8_bool_binop_done; } + __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L8_bool_binop_done:; + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":142 + /* "trunk/gensim/models/doc2vec_inner.pyx":135 + * f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: + * continue # <<<<<<<<<<<<<< + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + * g = (label - f) * alpha + */ + goto __pyx_L3_continue; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":136 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): + * g = (label - f) * alpha + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/doc2vec_inner.pyx":143 + /* "trunk/gensim/models/doc2vec_inner.pyx":137 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1[row2 + a] + * g = (label - f) * alpha # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); + __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":144 + /* "trunk/gensim/models/doc2vec_inner.pyx":138 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1[row2 + a] - * if tw: + * g = (label - f) * alpha + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":145 - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - * work[a] += g * syn1[row2 + a] # <<<<<<<<<<<<<< - * if tw: - * for a in range(size): + /* "trunk/gensim/models/doc2vec_inner.pyx":139 + * g = (label - f) * alpha + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if learn_context: */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); + __pyx_t_3 = (__pyx_v_learn_hidden != 0); + if (__pyx_t_3) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":140 + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * if learn_context: + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L10; } + __pyx_L10:; + __pyx_L3_continue:; + } - /* "trunk/gensim/models/doc2vec_inner.pyx":146 - * for a in range(size): - * work[a] += g * syn1[row2 + a] - * if tw: # <<<<<<<<<<<<<< - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":141 + * if learn_hidden: + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if learn_context: # <<<<<<<<<<<<<< + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) + * */ - __pyx_t_5 = (__pyx_v_tw != 0); - if (__pyx_t_5) { + __pyx_t_3 = (__pyx_v_learn_context != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":147 - * work[a] += g * syn1[row2 + a] - * if tw: - * for a in range(size): # <<<<<<<<<<<<<< - * syn1[row2 + a] += g * syn0[row1 + a] - * if tl: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; - - /* "trunk/gensim/models/doc2vec_inner.pyx":148 - * if tw: - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] # <<<<<<<<<<<<<< - * if tl: - * for a in range(size): - */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1[__pyx_t_7]) = ((__pyx_v_syn1[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]))); - } - goto __pyx_L14; - } - __pyx_L14:; - __pyx_L5_continue:; + /* "trunk/gensim/models/doc2vec_inner.pyx":142 + * our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + * if learn_context: + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) # <<<<<<<<<<<<<< + * + * return next_random + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_context_locks[__pyx_v_context_index])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_context_vectors[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L11; } + __pyx_L11:; - /* "trunk/gensim/models/doc2vec_inner.pyx":149 - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] - * if tl: # <<<<<<<<<<<<<< - * for a in range(size): - * syn0[row1 + a] += work[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":144 + * our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) + * + * return next_random # <<<<<<<<<<<<<< + * + * */ - __pyx_t_5 = (__pyx_v_tl != 0); - if (__pyx_t_5) { + __pyx_r = __pyx_v_next_random; + goto __pyx_L0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":107 + * + * + * cdef unsigned long long fast_document_dbow_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, + * REAL_t *context_vectors, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + */ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/doc2vec_inner.pyx":147 + * + * + * cdef void fast_document_dm_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + * REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, + */ + +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int __pyx_v_word_code_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int const __pyx_v_size, int __pyx_v_learn_hidden) { + PY_LONG_LONG __pyx_v_b; + PY_LONG_LONG __pyx_v_row2; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; + int __pyx_t_1; + PY_LONG_LONG __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "trunk/gensim/models/doc2vec_inner.pyx":158 + * # l1 already composed by caller, passed in as neu1 + * # work (also passed in) will accumulate l1 error + * for b in range(word_code_len): # <<<<<<<<<<<<<< + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) + */ + __pyx_t_1 = __pyx_v_word_code_len; + for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { + __pyx_v_b = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":159 + * # work (also passed in) will accumulate l1 error + * for b in range(word_code_len): + * row2 = word_point[b] * size # <<<<<<<<<<<<<< + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: + */ + __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); + + /* "trunk/gensim/models/doc2vec_inner.pyx":160 + * for b in range(word_code_len): + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< + * if f <= -MAX_EXP or f >= MAX_EXP: + * continue + */ + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + + /* "trunk/gensim/models/doc2vec_inner.pyx":161 + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< + * continue + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + */ + __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); + if (!__pyx_t_4) { + } else { + __pyx_t_3 = __pyx_t_4; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L6_bool_binop_done:; + if (__pyx_t_3) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":162 + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: + * continue # <<<<<<<<<<<<<< + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + * g = (1 - word_code[b] - f) * alpha + */ + goto __pyx_L3_continue; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":163 + * if f <= -MAX_EXP or f >= MAX_EXP: + * continue + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + */ + __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); + + /* "trunk/gensim/models/doc2vec_inner.pyx":164 + * continue + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: + */ + __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":150 - * syn1[row2 + a] += g * syn0[row1 + a] - * if tl: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[row1 + a] += work[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":165 + * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + + /* "trunk/gensim/models/doc2vec_inner.pyx":166 + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) * */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_3 = (__pyx_v_learn_hidden != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":151 - * if tl: - * for a in range(size): - * syn0[row1 + a] += work[a] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":167 + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * * */ - __pyx_t_4 = (__pyx_v_row1 + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_4]) = ((__pyx_v_syn0[__pyx_t_4]) + (__pyx_v_work[__pyx_v_a])); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L8; } - goto __pyx_L17; + __pyx_L8:; + __pyx_L3_continue:; } - __pyx_L17:; - /* "trunk/gensim/models/doc2vec_inner.pyx":124 + /* "trunk/gensim/models/doc2vec_inner.pyx":147 * * - * cdef void fast_sentence2_dbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * cdef void fast_document_dm_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + * REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, */ /* function exit code */ } -/* "trunk/gensim/models/doc2vec_inner.pyx":154 +/* "trunk/gensim/models/doc2vec_inner.pyx":170 * * - * cdef unsigned long long fast_sentence0_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * cdef unsigned long long fast_document_dm_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + * REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, */ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_row1; +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, unsigned PY_LONG_LONG __pyx_v_next_random, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_predict_word_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int const __pyx_v_size, int __pyx_v_learn_hidden) { PY_LONG_LONG __pyx_v_row2; unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; @@ -2028,66 +2231,48 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast int __pyx_t_3; int __pyx_t_4; - /* "trunk/gensim/models/doc2vec_inner.pyx":161 + /* "trunk/gensim/models/doc2vec_inner.pyx":176 * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":162 - * cdef long long a - * cdef long long row1 = word2_index * size, row2 + * cdef long long row2 * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< * cdef REAL_t f, g, label * cdef np.uint32_t target_index */ __pyx_v_modulo = 281474976710655ULL; - /* "trunk/gensim/models/doc2vec_inner.pyx":167 - * cdef int d - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":169 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * + /* "trunk/gensim/models/doc2vec_inner.pyx":183 + * # l1 already composed by caller, passed in as neu1 + * # work (also passsed in) will accumulate l1 error for outside application * for d in range(negative+1): # <<<<<<<<<<<<<< * if d == 0: - * target_index = word_index + * target_index = predict_word_index */ __pyx_t_1 = (__pyx_v_negative + 1); for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_d = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":170 - * + /* "trunk/gensim/models/doc2vec_inner.pyx":184 + * # work (also passsed in) will accumulate l1 error for outside application * for d in range(negative+1): * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index + * target_index = predict_word_index * label = ONEF */ __pyx_t_3 = ((__pyx_v_d == 0) != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":171 + /* "trunk/gensim/models/doc2vec_inner.pyx":185 * for d in range(negative+1): * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< + * target_index = predict_word_index # <<<<<<<<<<<<<< * label = ONEF * else: */ - __pyx_v_target_index = __pyx_v_word_index; + __pyx_v_target_index = __pyx_v_predict_word_index; - /* "trunk/gensim/models/doc2vec_inner.pyx":172 + /* "trunk/gensim/models/doc2vec_inner.pyx":186 * if d == 0: - * target_index = word_index + * target_index = predict_word_index * label = ONEF # <<<<<<<<<<<<<< * else: * target_index = table[(next_random >> 16) % table_len] @@ -2097,37 +2282,37 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":174 + /* "trunk/gensim/models/doc2vec_inner.pyx":188 * label = ONEF * else: * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: + * if target_index == predict_word_index: */ __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - /* "trunk/gensim/models/doc2vec_inner.pyx":175 + /* "trunk/gensim/models/doc2vec_inner.pyx":189 * else: * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: + * if target_index == predict_word_index: * continue */ __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - /* "trunk/gensim/models/doc2vec_inner.pyx":176 + /* "trunk/gensim/models/doc2vec_inner.pyx":190 * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< + * if target_index == predict_word_index: # <<<<<<<<<<<<<< * continue * label = 0.0 */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); + __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_predict_word_index) != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":177 + /* "trunk/gensim/models/doc2vec_inner.pyx":191 * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: + * if target_index == predict_word_index: * continue # <<<<<<<<<<<<<< * label = 0.0 * @@ -2135,8 +2320,8 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":178 - * if target_index == word_index: + /* "trunk/gensim/models/doc2vec_inner.pyx":192 + * if target_index == predict_word_index: * continue * label = 0.0 # <<<<<<<<<<<<<< * @@ -2146,27 +2331,27 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast } __pyx_L5:; - /* "trunk/gensim/models/doc2vec_inner.pyx":180 + /* "trunk/gensim/models/doc2vec_inner.pyx":194 * label = 0.0 * * row2 = target_index * size # <<<<<<<<<<<<<< - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":181 + /* "trunk/gensim/models/doc2vec_inner.pyx":195 * * row2 = target_index * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":182 + /* "trunk/gensim/models/doc2vec_inner.pyx":196 * row2 = target_index * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -2182,8 +2367,8 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast __pyx_L8_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":183 - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":197 + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -2192,95 +2377,73 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":184 + /* "trunk/gensim/models/doc2vec_inner.pyx":198 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/doc2vec_inner.pyx":185 + /* "trunk/gensim/models/doc2vec_inner.pyx":199 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: */ __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":186 + /* "trunk/gensim/models/doc2vec_inner.pyx":200 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":187 + /* "trunk/gensim/models/doc2vec_inner.pyx":201 * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * */ - __pyx_t_3 = (__pyx_v_tw != 0); + __pyx_t_3 = (__pyx_v_learn_hidden != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":188 - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":202 + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * + * return next_random */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); goto __pyx_L10; } __pyx_L10:; __pyx_L3_continue:; } - /* "trunk/gensim/models/doc2vec_inner.pyx":189 - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - * - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":190 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; - } - __pyx_L11:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":192 - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":204 + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) * * return next_random # <<<<<<<<<<<<<< * - * cdef unsigned long long fast_sentence1_dbow_neg( + * cdef void fast_document_dmc_hs( */ __pyx_r = __pyx_v_next_random; goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":154 + /* "trunk/gensim/models/doc2vec_inner.pyx":170 * * - * cdef unsigned long long fast_sentence0_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * cdef unsigned long long fast_document_dm_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + * REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, */ /* function exit code */ @@ -2288,168 +2451,56 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":194 +/* "trunk/gensim/models/doc2vec_inner.pyx":206 * return next_random * - * cdef unsigned long long fast_sentence1_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * cdef void fast_document_dmc_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + * REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, */ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_row1; +static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int __pyx_v_word_code_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int const __pyx_v_layer1_size, CYTHON_UNUSED int const __pyx_v_vector_size, int __pyx_v_learn_hidden) { + PY_LONG_LONG __pyx_v_b; PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - int __pyx_v_d; - unsigned PY_LONG_LONG __pyx_r; - long __pyx_t_1; - int __pyx_t_2; + int __pyx_t_1; + PY_LONG_LONG __pyx_t_2; int __pyx_t_3; int __pyx_t_4; - /* "trunk/gensim/models/doc2vec_inner.pyx":201 - * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":202 - * cdef long long a - * cdef long long row1 = word2_index * size, row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, label - * cdef np.uint32_t target_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/doc2vec_inner.pyx":207 - * cdef int d - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":209 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * - * if d == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":218 + * # l1 already composed by caller, passed in as neu1 + * # work accumulates net l1 error; eventually applied by caller + * for b in range(word_code_len): # <<<<<<<<<<<<<< + * row2 = word_point[b] * layer1_size + * f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) */ - __pyx_t_1 = (__pyx_v_negative + 1); + __pyx_t_1 = __pyx_v_word_code_len; for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_d = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":211 - * for d in range(negative+1): - * - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":212 - * - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/doc2vec_inner.pyx":213 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; - goto __pyx_L5; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":215 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":216 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/doc2vec_inner.pyx":217 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":218 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L3_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":219 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - __pyx_L5:; + __pyx_v_b = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":221 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":219 + * # work accumulates net l1 error; eventually applied by caller + * for b in range(word_code_len): + * row2 = word_point[b] * layer1_size # <<<<<<<<<<<<<< + * f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); + __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_layer1_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":222 - * - * row2 = target_index * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":220 + * for b in range(word_code_len): + * row2 = word_point[b] * layer1_size + * f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_layer1_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":223 - * row2 = target_index * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":221 + * row2 = word_point[b] * layer1_size + * f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -2458,130 +2509,94 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast if (!__pyx_t_4) { } else { __pyx_t_3 = __pyx_t_4; - goto __pyx_L8_bool_binop_done; + goto __pyx_L6_bool_binop_done; } __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); __pyx_t_3 = __pyx_t_4; - __pyx_L8_bool_binop_done:; + __pyx_L6_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":224 - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":222 + * f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha + * g = (1 - word_code[b] - f) * alpha */ goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":225 + /* "trunk/gensim/models/doc2vec_inner.pyx":223 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/doc2vec_inner.pyx":226 + /* "trunk/gensim/models/doc2vec_inner.pyx":224 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: + * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< + * our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); + __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":227 + /* "trunk/gensim/models/doc2vec_inner.pyx":225 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1[row2], &ONE) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_layer1_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":228 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: + /* "trunk/gensim/models/doc2vec_inner.pyx":226 + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1[row2], &ONE) + * */ - __pyx_t_3 = (__pyx_v_tw != 0); + __pyx_t_3 = (__pyx_v_learn_hidden != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":229 - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L10; - } - __pyx_L10:; - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":230 - * if tw: - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":227 + * our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":231 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if tl: - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< * - * return next_random */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_layer1_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L8; + } + __pyx_L8:; + __pyx_L3_continue:; } - __pyx_L11:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":233 - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence2_dbow_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":194 + /* "trunk/gensim/models/doc2vec_inner.pyx":206 * return next_random * - * cdef unsigned long long fast_sentence1_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * cdef void fast_document_dmc_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + * REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, */ /* function exit code */ - __pyx_L0:; - return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":235 - * return next_random +/* "trunk/gensim/models/doc2vec_inner.pyx":230 * - * cdef unsigned long long fast_sentence2_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * + * cdef unsigned long long fast_document_dmc_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + * REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, */ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_row1; +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, unsigned PY_LONG_LONG __pyx_v_next_random, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_predict_word_index, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int const __pyx_v_layer1_size, CYTHON_UNUSED int const __pyx_v_vector_size, int __pyx_v_learn_hidden) { PY_LONG_LONG __pyx_v_row2; unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; @@ -2590,339 +2605,224 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast __pyx_t_5numpy_uint32_t __pyx_v_target_index; int __pyx_v_d; unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - long __pyx_t_3; + long __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - - /* "trunk/gensim/models/doc2vec_inner.pyx":242 - * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":243 + /* "trunk/gensim/models/doc2vec_inner.pyx":237 * cdef long long a - * cdef long long row1 = word2_index * size, row2 + * cdef long long row2 * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< * cdef REAL_t f, g, label * cdef np.uint32_t target_index */ __pyx_v_modulo = 281474976710655ULL; - /* "trunk/gensim/models/doc2vec_inner.pyx":248 - * cdef int d - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":249 - * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":251 - * work[a] = 0.0 - * + /* "trunk/gensim/models/doc2vec_inner.pyx":244 + * # l1 already composed by caller, passed in as neu1 + * # work accumulates net l1 error; eventually applied by caller * for d in range(negative+1): # <<<<<<<<<<<<<< - * * if d == 0: + * target_index = predict_word_index */ - __pyx_t_3 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_3; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; + __pyx_t_1 = (__pyx_v_negative + 1); + for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { + __pyx_v_d = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":253 + /* "trunk/gensim/models/doc2vec_inner.pyx":245 + * # work accumulates net l1 error; eventually applied by caller * for d in range(negative+1): - * * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index + * target_index = predict_word_index * label = ONEF */ - __pyx_t_4 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_4) { + __pyx_t_3 = ((__pyx_v_d == 0) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":254 - * + /* "trunk/gensim/models/doc2vec_inner.pyx":246 + * for d in range(negative+1): * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< + * target_index = predict_word_index # <<<<<<<<<<<<<< * label = ONEF * else: */ - __pyx_v_target_index = __pyx_v_word_index; + __pyx_v_target_index = __pyx_v_predict_word_index; - /* "trunk/gensim/models/doc2vec_inner.pyx":255 + /* "trunk/gensim/models/doc2vec_inner.pyx":247 * if d == 0: - * target_index = word_index + * target_index = predict_word_index * label = ONEF # <<<<<<<<<<<<<< * else: * target_index = table[(next_random >> 16) % table_len] */ __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; - goto __pyx_L7; + goto __pyx_L5; } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":257 + /* "trunk/gensim/models/doc2vec_inner.pyx":249 * label = ONEF * else: * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: + * if target_index == predict_word_index: */ __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - /* "trunk/gensim/models/doc2vec_inner.pyx":258 + /* "trunk/gensim/models/doc2vec_inner.pyx":250 * else: * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: + * if target_index == predict_word_index: * continue */ __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - /* "trunk/gensim/models/doc2vec_inner.pyx":259 + /* "trunk/gensim/models/doc2vec_inner.pyx":251 * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< + * if target_index == predict_word_index: # <<<<<<<<<<<<<< * continue * label = 0.0 */ - __pyx_t_4 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_4) { + __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_predict_word_index) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":260 + /* "trunk/gensim/models/doc2vec_inner.pyx":252 * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: + * if target_index == predict_word_index: * continue # <<<<<<<<<<<<<< * label = 0.0 * */ - goto __pyx_L5_continue; + goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":261 - * if target_index == word_index: + /* "trunk/gensim/models/doc2vec_inner.pyx":253 + * if target_index == predict_word_index: * continue * label = 0.0 # <<<<<<<<<<<<<< * - * row2 = target_index * size + * row2 = target_index * layer1_size */ __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); } - __pyx_L7:; + __pyx_L5:; - /* "trunk/gensim/models/doc2vec_inner.pyx":263 + /* "trunk/gensim/models/doc2vec_inner.pyx":255 * label = 0.0 * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":264 - * - * row2 = target_index * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":265 - * row2 = target_index * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += syn0[row1 + a] * syn1neg[row2 + a] + * row2 = target_index * layer1_size # <<<<<<<<<<<<<< + * f = our_dot(&layer1_size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_layer1_size); - /* "trunk/gensim/models/doc2vec_inner.pyx":266 - * f = 0.0 - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":256 + * + * row2 = target_index * layer1_size + * f = our_dot(&layer1_size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]) * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot((&__pyx_v_layer1_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":267 - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":257 + * row2 = target_index * layer1_size + * f = our_dot(&layer1_size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] */ - __pyx_t_6 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_6) { + __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); + if (!__pyx_t_4) { } else { - __pyx_t_4 = __pyx_t_6; - goto __pyx_L12_bool_binop_done; + __pyx_t_3 = __pyx_t_4; + goto __pyx_L8_bool_binop_done; } - __pyx_t_6 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_6; - __pyx_L12_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L8_bool_binop_done:; + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":268 - * f += syn0[row1 + a] * syn1neg[row2 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":258 + * f = our_dot(&layer1_size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha */ - goto __pyx_L5_continue; + goto __pyx_L3_continue; } - /* "trunk/gensim/models/doc2vec_inner.pyx":269 + /* "trunk/gensim/models/doc2vec_inner.pyx":259 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (label - f) * alpha - * for a in range(size): + * our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/doc2vec_inner.pyx":270 + /* "trunk/gensim/models/doc2vec_inner.pyx":260 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] + * our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: */ __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/doc2vec_inner.pyx":271 + /* "trunk/gensim/models/doc2vec_inner.pyx":261 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1neg[row2 + a] - * if tw: + * our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * if learn_hidden: + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1neg[row2], &ONE) */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_layer1_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":272 + /* "trunk/gensim/models/doc2vec_inner.pyx":262 * g = (label - f) * alpha - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * if tw: - * for a in range(size): + * our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: # <<<<<<<<<<<<<< + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } + __pyx_t_3 = (__pyx_v_learn_hidden != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":273 - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - * if tw: # <<<<<<<<<<<<<< - * for a in range(size): - * syn1neg[row2 + a] += g * syn0[row1 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":263 + * our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) + * if learn_hidden: + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * + * return next_random */ - __pyx_t_4 = (__pyx_v_tw != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":274 - * work[a] += g * syn1neg[row2 + a] - * if tw: - * for a in range(size): # <<<<<<<<<<<<<< - * syn1neg[row2 + a] += g * syn0[row1 + a] - * if tl: - */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":275 - * if tw: - * for a in range(size): - * syn1neg[row2 + a] += g * syn0[row1 + a] # <<<<<<<<<<<<<< - * if tl: - * for a in range(size): - */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1neg[__pyx_t_7]) = ((__pyx_v_syn1neg[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]))); - } - goto __pyx_L16; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_layer1_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L10; } - __pyx_L16:; - __pyx_L5_continue:; + __pyx_L10:; + __pyx_L3_continue:; } - /* "trunk/gensim/models/doc2vec_inner.pyx":276 - * for a in range(size): - * syn1neg[row2 + a] += g * syn0[row1 + a] - * if tl: # <<<<<<<<<<<<<< - * for a in range(size): - * syn0[row1 + a] += work[a] - */ - __pyx_t_4 = (__pyx_v_tl != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":277 - * syn1neg[row2 + a] += g * syn0[row1 + a] - * if tl: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[row1 + a] += work[a] - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":278 - * if tl: - * for a in range(size): - * syn0[row1 + a] += work[a] # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_t_7 = (__pyx_v_row1 + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - goto __pyx_L19; - } - __pyx_L19:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":280 - * syn0[row1 + a] += work[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":265 + * our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1neg[row2], &ONE) * * return next_random # <<<<<<<<<<<<<< * - * cdef void fast_sentence0_dm_hs( + * */ __pyx_r = __pyx_v_next_random; goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":235 - * return next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":230 * - * cdef unsigned long long fast_sentence2_dbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + * + * cdef unsigned long long fast_document_dmc_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + * REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, */ /* function exit code */ @@ -2930,3301 +2830,1438 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":282 - * return next_random +/* "trunk/gensim/models/doc2vec_inner.pyx":268 * - * cdef void fast_sentence0_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5numpy_uint32_t const *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_inv_count; - int __pyx_v_m; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - PY_LONG_LONG __pyx_t_5; - - /* "trunk/gensim/models/doc2vec_inner.pyx":293 - * cdef int m * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, # <<<<<<<<<<<<<< + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - /* "trunk/gensim/models/doc2vec_inner.pyx":294 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); +/* Python wrapper */ +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_document_dbow(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_1train_document_dbow = {"train_document_dbow", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_document_dbow, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_document_dbow(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_model = 0; + PyObject *__pyx_v_word_vocabs = 0; + PyObject *__pyx_v_doctag_indexes = 0; + PyObject *__pyx_v_alpha = 0; + PyObject *__pyx_v_work = 0; + PyObject *__pyx_v_train_words = 0; + PyObject *__pyx_v_learn_doctags = 0; + PyObject *__pyx_v_learn_words = 0; + PyObject *__pyx_v_learn_hidden = 0; + PyObject *__pyx_v_word_vectors = 0; + PyObject *__pyx_v_word_locks = 0; + PyObject *__pyx_v_doctag_vectors = 0; + PyObject *__pyx_v_doctag_locks = 0; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("train_document_dbow (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_model,&__pyx_n_s_word_vocabs,&__pyx_n_s_doctag_indexes,&__pyx_n_s_alpha,&__pyx_n_s_work,&__pyx_n_s_train_words,&__pyx_n_s_learn_doctags,&__pyx_n_s_learn_words,&__pyx_n_s_learn_hidden,&__pyx_n_s_word_vectors,&__pyx_n_s_word_locks,&__pyx_n_s_doctag_vectors,&__pyx_n_s_doctag_locks,0}; + PyObject* values[13] = {0,0,0,0,0,0,0,0,0,0,0,0,0}; + values[4] = ((PyObject *)Py_None); - /* "trunk/gensim/models/doc2vec_inner.pyx":295 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":269 + * + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, # <<<<<<<<<<<<<< + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + * cdef int hs = model.hs */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; + values[5] = ((PyObject *)Py_False); + values[6] = ((PyObject *)Py_True); + values[7] = ((PyObject *)Py_True); + values[8] = ((PyObject *)Py_True); - /* "trunk/gensim/models/doc2vec_inner.pyx":296 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: + /* "trunk/gensim/models/doc2vec_inner.pyx":270 + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): # <<<<<<<<<<<<<< + * cdef int hs = model.hs + * cdef int negative = model.negative */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { + values[9] = ((PyObject *)Py_None); + values[10] = ((PyObject *)Py_None); + values[11] = ((PyObject *)Py_None); + values[12] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_model)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vocabs)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("train_document_dbow", 0, 4, 13, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_indexes)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("train_document_dbow", 0, 4, 13, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + case 3: + if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_alpha)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("train_document_dbow", 0, 4, 13, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + case 4: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work); + if (value) { values[4] = value; kw_args--; } + } + case 5: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_train_words); + if (value) { values[5] = value; kw_args--; } + } + case 6: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_doctags); + if (value) { values[6] = value; kw_args--; } + } + case 7: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_words); + if (value) { values[7] = value; kw_args--; } + } + case 8: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_hidden); + if (value) { values[8] = value; kw_args--; } + } + case 9: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vectors); + if (value) { values[9] = value; kw_args--; } + } + case 10: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_locks); + if (value) { values[10] = value; kw_args--; } + } + case 11: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_vectors); + if (value) { values[11] = value; kw_args--; } + } + case 12: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_locks); + if (value) { values[12] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_document_dbow") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { + __pyx_v_model = values[0]; + __pyx_v_word_vocabs = values[1]; + __pyx_v_doctag_indexes = values[2]; + __pyx_v_alpha = values[3]; + __pyx_v_work = values[4]; + __pyx_v_train_words = values[5]; + __pyx_v_learn_doctags = values[6]; + __pyx_v_learn_words = values[7]; + __pyx_v_learn_hidden = values[8]; + __pyx_v_word_vectors = values[9]; + __pyx_v_word_locks = values[10]; + __pyx_v_doctag_vectors = values[11]; + __pyx_v_doctag_locks = values[12]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("train_document_dbow", 0, 4, 13, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dbow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_document_dbow(__pyx_self, __pyx_v_model, __pyx_v_word_vocabs, __pyx_v_doctag_indexes, __pyx_v_alpha, __pyx_v_work, __pyx_v_train_words, __pyx_v_learn_doctags, __pyx_v_learn_words, __pyx_v_learn_hidden, __pyx_v_word_vectors, __pyx_v_word_locks, __pyx_v_doctag_vectors, __pyx_v_doctag_locks); - /* "trunk/gensim/models/doc2vec_inner.pyx":297 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF + /* "trunk/gensim/models/doc2vec_inner.pyx":268 + * + * + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, # <<<<<<<<<<<<<< + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - goto __pyx_L3_continue; - } - /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":299 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} - /* "trunk/gensim/models/doc2vec_inner.pyx":300 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_document_dbow(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_train_words, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks) { + int __pyx_v_hs; + int __pyx_v_negative; + int __pyx_v__train_words; + int __pyx_v__learn_words; + int __pyx_v__learn_hidden; + int __pyx_v__learn_doctags; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__work; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v__alpha; + int __pyx_v_size; + int __pyx_v_codelens[10000]; + __pyx_t_5numpy_uint32_t __pyx_v_indexes[10000]; + __pyx_t_5numpy_uint32_t __pyx_v__doctag_indexes[10000]; + __pyx_t_5numpy_uint32_t __pyx_v_reduced_windows[10000]; + int __pyx_v_document_len; + int __pyx_v_doctag_len; + int __pyx_v_window; + int __pyx_v_i; + int __pyx_v_j; + long __pyx_v_result; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1; + __pyx_t_5numpy_uint32_t *__pyx_v_points[10000]; + __pyx_t_5numpy_uint8_t *__pyx_v_codes[10000]; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg; + __pyx_t_5numpy_uint32_t *__pyx_v_table; + unsigned PY_LONG_LONG __pyx_v_table_len; + unsigned PY_LONG_LONG __pyx_v_next_random; + PyObject *__pyx_v_predict_word = NULL; + PyObject *__pyx_v_item = NULL; + long __pyx_v_k; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + unsigned PY_LONG_LONG __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + long __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_t_13; + __pyx_t_5numpy_uint32_t __pyx_t_14; + PyObject *__pyx_t_15 = NULL; + PyObject *__pyx_t_16 = NULL; + PyObject *(*__pyx_t_17)(PyObject *); + int __pyx_t_18; + int __pyx_t_19; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("train_document_dbow", 0); + __Pyx_INCREF(__pyx_v_work); + __Pyx_INCREF(__pyx_v_word_vectors); + __Pyx_INCREF(__pyx_v_word_locks); + __Pyx_INCREF(__pyx_v_doctag_vectors); + __Pyx_INCREF(__pyx_v_doctag_locks); + + /* "trunk/gensim/models/doc2vec_inner.pyx":271 + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + * cdef int hs = model.hs # <<<<<<<<<<<<<< + * cdef int negative = model.negative + * cdef int _train_words = train_words */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L3_continue:; - } + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 271; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 271; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_hs = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":301 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":272 + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + * cdef int hs = model.hs + * cdef int negative = model.negative # <<<<<<<<<<<<<< + * cdef int _train_words = train_words + * cdef int _learn_words = learn_words */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 272; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 272; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_negative = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":302 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: + /* "trunk/gensim/models/doc2vec_inner.pyx":273 + * cdef int hs = model.hs + * cdef int negative = model.negative + * cdef int _train_words = train_words # <<<<<<<<<<<<<< + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_train_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 273; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__train_words = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":303 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF + /* "trunk/gensim/models/doc2vec_inner.pyx":274 + * cdef int negative = model.negative + * cdef int _train_words = train_words + * cdef int _learn_words = learn_words # <<<<<<<<<<<<<< + * cdef int _learn_hidden = learn_hidden + * cdef int _learn_doctags = learn_doctags */ - goto __pyx_L8_continue; - } - /*else*/ { + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 274; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_words = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":305 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":275 + * cdef int _train_words = train_words + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden # <<<<<<<<<<<<<< + * cdef int _learn_doctags = learn_doctags * */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_hidden); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 275; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_hidden = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":306 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":276 + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden + * cdef int _learn_doctags = learn_doctags # <<<<<<<<<<<<<< * - * if cbow_mean and count > (0.5): + * cdef REAL_t *_word_vectors */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L8_continue:; - } + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_doctags); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 276; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_doctags = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":308 - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":283 + * cdef REAL_t *_doctag_locks + * cdef REAL_t *_work + * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< + * cdef int size = model.layer1_size * - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L12_bool_binop_done:; - if (__pyx_t_3) { + __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 283; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__alpha = __pyx_t_3; - /* "trunk/gensim/models/doc2vec_inner.pyx":309 - * - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":284 + * cdef REAL_t *_work + * cdef REAL_t _alpha = alpha + * cdef int size = model.layer1_size # <<<<<<<<<<<<<< * + * cdef int codelens[MAX_DOCUMENT_LEN] */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF / __pyx_v_count); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 284; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 284; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_size = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":310 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":292 + * cdef int document_len + * cdef int doctag_len + * cdef int window = model.window # <<<<<<<<<<<<<< * - * memset(work, 0, size * cython.sizeof(REAL_t)) + * cdef int i, j */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; - } - __pyx_L11:; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 292; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 292; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_window = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":312 - * sscal(&size, &inv_count, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":295 * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size + * cdef int i, j + * cdef long result = 0 # <<<<<<<<<<<<<< + * + * # For hierarchical softmax */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); + __pyx_v_result = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":313 + /* "trunk/gensim/models/doc2vec_inner.pyx":309 * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: # <<<<<<<<<<<<<< + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_1; __pyx_t_5+=1) { - __pyx_v_b = __pyx_t_5; + __pyx_t_4 = (__pyx_v_word_vectors == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":314 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: + /* "trunk/gensim/models/doc2vec_inner.pyx":310 + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: + * word_vectors = model.syn0 # <<<<<<<<<<<<<< + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 310; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_word_vectors, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L3; + } + __pyx_L3:; - /* "trunk/gensim/models/doc2vec_inner.pyx":315 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":311 + * if word_vectors is None: + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) # <<<<<<<<<<<<<< + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); + if (!(likely(((__pyx_v_word_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 311; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_vectors))); - /* "trunk/gensim/models/doc2vec_inner.pyx":316 - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L17_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L17_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":317 - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":318 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":319 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":320 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - - /* "trunk/gensim/models/doc2vec_inner.pyx":321 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { + /* "trunk/gensim/models/doc2vec_inner.pyx":312 + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: # <<<<<<<<<<<<<< + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + */ + __pyx_t_5 = (__pyx_v_doctag_vectors == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":322 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if tw: - * for m in range(j, k): + /* "trunk/gensim/models/doc2vec_inner.pyx":313 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 # <<<<<<<<<<<<<< + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L19; - } - __pyx_L19:; - __pyx_L14_continue:; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 313; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_doctag_syn0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 313; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_vectors, __pyx_t_6); + __pyx_t_6 = 0; + goto __pyx_L4; } + __pyx_L4:; - /* "trunk/gensim/models/doc2vec_inner.pyx":323 - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":324 - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":325 - * if tw: - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L24_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L24_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":326 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - */ - goto __pyx_L21_continue; - } - /*else*/ { + /* "trunk/gensim/models/doc2vec_inner.pyx":314 + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) # <<<<<<<<<<<<<< + * if word_locks is None: + * word_locks = model.syn0_lockf + */ + if (!(likely(((__pyx_v_doctag_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 314; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_vectors))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":315 + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: # <<<<<<<<<<<<<< + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + */ + __pyx_t_4 = (__pyx_v_word_locks == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":328 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): + /* "trunk/gensim/models/doc2vec_inner.pyx":316 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: + * word_locks = model.syn0_lockf # <<<<<<<<<<<<<< + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L21_continue:; - } - goto __pyx_L20; + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0_lockf); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 316; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF_SET(__pyx_v_word_locks, __pyx_t_6); + __pyx_t_6 = 0; + goto __pyx_L5; } - __pyx_L20:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":329 - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":330 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":331 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":332 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - */ - goto __pyx_L27_continue; - } - /*else*/ { + __pyx_L5:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":317 + * if word_locks is None: + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) # <<<<<<<<<<<<<< + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + */ + if (!(likely(((__pyx_v_word_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 317; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_locks))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":318 + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: # <<<<<<<<<<<<<< + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) + */ + __pyx_t_5 = (__pyx_v_doctag_locks == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":334 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":319 + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf # <<<<<<<<<<<<<< + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) * - * cdef void fast_sentence1_dm_hs( */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L27_continue:; - } - goto __pyx_L26; + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 319; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_doctag_syn0_lockf); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 319; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_locks, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L6; } - __pyx_L26:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":282 - * return next_random - * - * cdef void fast_sentence0_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} + __pyx_L6:; -/* "trunk/gensim/models/doc2vec_inner.pyx":336 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":320 + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) # <<<<<<<<<<<<<< * - * cdef void fast_sentence1_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, + * if hs: */ + if (!(likely(((__pyx_v_doctag_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 320; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_locks))); -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5numpy_uint32_t const *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_inv_count; - int __pyx_v_m; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - PY_LONG_LONG __pyx_t_5; - - /* "trunk/gensim/models/doc2vec_inner.pyx":347 - * cdef int m + /* "trunk/gensim/models/doc2vec_inner.pyx":322 + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":348 + * if hs: # <<<<<<<<<<<<<< + * syn1 = (np.PyArray_DATA(model.syn1)) * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":349 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":350 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":351 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":353 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":354 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":355 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":356 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":357 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF */ - goto __pyx_L8_continue; - } - /*else*/ { + __pyx_t_4 = (__pyx_v_hs != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":359 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":323 * - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":360 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< + * if hs: + * syn1 = (np.PyArray_DATA(model.syn1)) # <<<<<<<<<<<<<< * - * if cbow_mean and count > (0.5): + * if negative: */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L8_continue:; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 323; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 323; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_syn1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L7; } + __pyx_L7:; - /* "trunk/gensim/models/doc2vec_inner.pyx":362 - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":325 + * syn1 = (np.PyArray_DATA(model.syn1)) * - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count , neu1, &ONE) + * if negative: # <<<<<<<<<<<<<< + * syn1neg = (np.PyArray_DATA(model.syn1neg)) + * table = (np.PyArray_DATA(model.table)) */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); + __pyx_t_4 = (__pyx_v_negative != 0); if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L12_bool_binop_done:; - if (__pyx_t_3) { - /* "trunk/gensim/models/doc2vec_inner.pyx":363 - * - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count , neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":326 * + * if negative: + * syn1neg = (np.PyArray_DATA(model.syn1neg)) # <<<<<<<<<<<<<< + * table = (np.PyArray_DATA(model.table)) + * table_len = len(model.table) */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF / __pyx_v_count); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 326; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 326; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_syn1neg = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":364 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count , neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) + /* "trunk/gensim/models/doc2vec_inner.pyx":327 + * if negative: + * syn1neg = (np.PyArray_DATA(model.syn1neg)) + * table = (np.PyArray_DATA(model.table)) # <<<<<<<<<<<<<< + * table_len = len(model.table) + * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; - } - __pyx_L11:; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 327; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 327; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_table = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":366 - * sscal(&size, &inv_count , neu1, &ONE) + /* "trunk/gensim/models/doc2vec_inner.pyx":328 + * syn1neg = (np.PyArray_DATA(model.syn1neg)) + * table = (np.PyArray_DATA(model.table)) + * table_len = len(model.table) # <<<<<<<<<<<<<< + * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 328; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 328; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_table_len = __pyx_t_7; - /* "trunk/gensim/models/doc2vec_inner.pyx":367 + /* "trunk/gensim/models/doc2vec_inner.pyx":329 + * table = (np.PyArray_DATA(model.table)) + * table_len = len(model.table) + * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_1; __pyx_t_5+=1) { - __pyx_v_b = __pyx_t_5; - - /* "trunk/gensim/models/doc2vec_inner.pyx":368 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":369 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":370 - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L17_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L17_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":371 - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":372 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":373 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":374 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - - /* "trunk/gensim/models/doc2vec_inner.pyx":375 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":376 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if tw: - * for m in range(j, k): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L19; - } - __pyx_L19:; - __pyx_L14_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":377 - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":378 - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * if tw: - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":379 - * if tw: - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L24_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L24_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":380 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - */ - goto __pyx_L21_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":382 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L21_continue:; - } - goto __pyx_L20; - } - __pyx_L20:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":383 - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":384 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":385 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":386 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - */ - goto __pyx_L27_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":388 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * - * cdef void fast_sentence2_dm_hs( - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L27_continue:; - } - goto __pyx_L26; - } - __pyx_L26:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":336 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - * - * cdef void fast_sentence1_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/doc2vec_inner.pyx":390 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - * - * cdef void fast_sentence2_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - -static void __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5numpy_uint32_t const *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - int __pyx_v_m; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - PY_LONG_LONG __pyx_t_8; - - /* "trunk/gensim/models/doc2vec_inner.pyx":401 - * cdef int m - * - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] = 0.0 - * count = 0.0 - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":402 - * - * for a in range(size): - * neu1[a] = 0.0 # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - (__pyx_v_neu1[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":403 - * for a in range(size): - * neu1[a] = 0.0 - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":404 - * neu1[a] = 0.0 - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":405 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L8_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":406 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L5_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":408 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":409 - * else: - * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":410 - * count += ONEF - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L5_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":411 - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":412 - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":413 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L12_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":415 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[lbl_indexes[m] * size + a] - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":416 - * else: - * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[lbl_indexes[m] * size + a] - * - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":417 - * count += ONEF - * for a in range(size): - * neu1[a] += syn0[lbl_indexes[m] * size + a] # <<<<<<<<<<<<<< - * - * if cbow_mean and count > (0.5): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L12_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":419 - * neu1[a] += syn0[lbl_indexes[m] * size + a] - * - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] /= count - */ - __pyx_t_5 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L18_bool_binop_done; - } - __pyx_t_5 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L18_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":420 - * - * if cbow_mean and count > (0.5): - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] /= count - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":421 - * if cbow_mean and count > (0.5): - * for a in range(size): - * neu1[a] /= count # <<<<<<<<<<<<<< - * - * for a in range(size): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) / __pyx_v_count); - } - goto __pyx_L17; - } - __pyx_L17:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":423 - * neu1[a] /= count - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * for b in range(codelens[i]): - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":424 - * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size - */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":425 - * for a in range(size): - * work[a] = 0.0 - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = 0.0 - */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":426 - * work[a] = 0.0 - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":427 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":428 - * row2 = word_point[b] * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/doc2vec_inner.pyx":429 - * f = 0.0 - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_neu1[__pyx_v_a]) * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":430 - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_5 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L29_bool_binop_done; - } - __pyx_t_5 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L29_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":431 - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L24_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":432 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":433 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1[row2 + a] - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":434 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1[row2 + a] - * if tw: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/doc2vec_inner.pyx":435 - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - * work[a] += g * syn1[row2 + a] # <<<<<<<<<<<<<< - * if tw: - * for a in range(size): - */ - __pyx_t_8 = __pyx_v_a; - (__pyx_v_work[__pyx_t_8]) = ((__pyx_v_work[__pyx_t_8]) + (__pyx_v_g * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":436 - * for a in range(size): - * work[a] += g * syn1[row2 + a] - * if tw: # <<<<<<<<<<<<<< - * for a in range(size): - * syn1[row2 + a] += g * neu1[a] - */ - __pyx_t_4 = (__pyx_v_tw != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":437 - * work[a] += g * syn1[row2 + a] - * if tw: - * for a in range(size): # <<<<<<<<<<<<<< - * syn1[row2 + a] += g * neu1[a] - * if tw: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/doc2vec_inner.pyx":438 - * if tw: - * for a in range(size): - * syn1[row2 + a] += g * neu1[a] # <<<<<<<<<<<<<< - * if tw: - * for m in range(j, k): - */ - __pyx_t_8 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1[__pyx_t_8]) = ((__pyx_v_syn1[__pyx_t_8]) + (__pyx_v_g * (__pyx_v_neu1[__pyx_v_a]))); - } - goto __pyx_L33; - } - __pyx_L33:; - __pyx_L24_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":439 - * for a in range(size): - * syn1[row2 + a] += g * neu1[a] - * if tw: # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_t_4 = (__pyx_v_tw != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":440 - * syn1[row2 + a] += g * neu1[a] - * if tw: - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":441 - * if tw: - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L40_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L40_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":442 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * for a in range(size): - */ - goto __pyx_L37_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":444 - * continue - * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[indexes[m] * size + a] += work[a] - * if tl: - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":445 - * else: - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): - */ - __pyx_t_7 = (((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - } - __pyx_L37_continue:; - } - goto __pyx_L36; - } - __pyx_L36:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":446 - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_4 = (__pyx_v_tl != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":447 - * syn0[indexes[m] * size + a] += work[a] - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":448 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":449 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * for a in range(size): - */ - goto __pyx_L45_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":451 - * continue - * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[lbl_indexes[m] * size + a] += work[a] - * - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":452 - * else: - * for a in range(size): - * syn0[lbl_indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence0_dm_neg( - */ - __pyx_t_7 = (((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - } - __pyx_L45_continue:; - } - goto __pyx_L44; - } - __pyx_L44:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":390 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - * - * cdef void fast_sentence2_dm_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/doc2vec_inner.pyx":454 - * syn0[lbl_indexes[m] * size + a] += work[a] - * - * cdef unsigned long long fast_sentence0_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t *__pyx_v_indexes, __pyx_t_5numpy_uint32_t *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_inv_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - __pyx_t_5numpy_uint32_t __pyx_v_word_index; - int __pyx_v_d; - int __pyx_v_m; - unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - long __pyx_t_5; - - /* "trunk/gensim/models/doc2vec_inner.pyx":462 - * cdef long long a - * cdef long long row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label - * cdef np.uint32_t target_index, word_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/doc2vec_inner.pyx":467 - * cdef int d, m - * - * word_index = indexes[i] # <<<<<<<<<<<<<< - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":469 - * word_index = indexes[i] - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":470 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":471 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":472 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":473 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":475 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":476 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":477 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":478 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":479 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L8_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":481 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":482 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L8_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":483 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) - */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L12_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":484 - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/doc2vec_inner.pyx":485 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; - } - __pyx_L11:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":487 - * sscal(&size, &inv_count, neu1, &ONE) - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":489 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index - */ - __pyx_t_5 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_5; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; - - /* "trunk/gensim/models/doc2vec_inner.pyx":490 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":491 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/doc2vec_inner.pyx":492 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; - goto __pyx_L16; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":494 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":495 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/doc2vec_inner.pyx":496 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":497 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":498 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - __pyx_L16:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":500 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":501 - * - * row2 = target_index * size - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":502 - * row2 = target_index * size - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L19_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L19_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":503 - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":504 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":505 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":506 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - - /* "trunk/gensim/models/doc2vec_inner.pyx":507 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":508 - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if tw: - * for m in range(j,k): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L21; - } - __pyx_L21:; - __pyx_L14_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":509 - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: # <<<<<<<<<<<<<< - * for m in range(j,k): - * if m == i or codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":510 - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: - * for m in range(j,k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":511 - * if tw: - * for m in range(j,k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L26_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L26_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":512 - * for m in range(j,k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - */ - goto __pyx_L23_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":514 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L23_continue:; - } - goto __pyx_L22; - } - __pyx_L22:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":515 - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":516 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":517 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":518 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - */ - goto __pyx_L29_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":520 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L29_continue:; - } - goto __pyx_L28; - } - __pyx_L28:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":522 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence1_dm_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/doc2vec_inner.pyx":454 - * syn0[lbl_indexes[m] * size + a] += work[a] - * - * cdef unsigned long long fast_sentence0_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - - /* function exit code */ - __pyx_L0:; - return __pyx_r; -} - -/* "trunk/gensim/models/doc2vec_inner.pyx":524 - * return next_random - * - * cdef unsigned long long fast_sentence1_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t *__pyx_v_indexes, __pyx_t_5numpy_uint32_t *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_inv_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - __pyx_t_5numpy_uint32_t __pyx_v_word_index; - int __pyx_v_d; - int __pyx_v_m; - unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - long __pyx_t_5; - - /* "trunk/gensim/models/doc2vec_inner.pyx":532 - * cdef long long a - * cdef long long row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label - * cdef np.uint32_t target_index, word_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/doc2vec_inner.pyx":537 - * cdef int d, m - * - * word_index = indexes[i] # <<<<<<<<<<<<<< - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":539 - * word_index = indexes[i] - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":540 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":541 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":542 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":543 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":545 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":546 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":547 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":548 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":549 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L8_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":551 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":552 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L8_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":553 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) - */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L12_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":554 - * saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/doc2vec_inner.pyx":555 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L11; - } - __pyx_L11:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":557 - * sscal(&size, &inv_count, neu1, &ONE) - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":559 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index - */ - __pyx_t_5 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_5; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; - - /* "trunk/gensim/models/doc2vec_inner.pyx":560 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":561 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/doc2vec_inner.pyx":562 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; - goto __pyx_L16; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":564 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":565 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/doc2vec_inner.pyx":566 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":567 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":568 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - __pyx_L16:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":570 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/doc2vec_inner.pyx":571 - * - * row2 = target_index * size - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE))); - - /* "trunk/gensim/models/doc2vec_inner.pyx":572 - * row2 = target_index * size - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L19_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L19_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":573 - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L14_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":574 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":575 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":576 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - - /* "trunk/gensim/models/doc2vec_inner.pyx":577 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":578 - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if tw: - * for m in range(j,k): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - goto __pyx_L21; - } - __pyx_L21:; - __pyx_L14_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":579 - * if tw: - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: # <<<<<<<<<<<<<< - * for m in range(j,k): - * if m == i or codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tw != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":580 - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * if tw: - * for m in range(j,k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":581 - * if tw: - * for m in range(j,k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L26_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L26_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":582 - * for m in range(j,k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - */ - goto __pyx_L23_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":584 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L23_continue:; - } - goto __pyx_L22; - } - __pyx_L22:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":585 - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_3 = (__pyx_v_tl != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":586 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":587 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_3 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":588 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - */ - goto __pyx_L29_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":590 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - } - __pyx_L29_continue:; - } - goto __pyx_L28; - } - __pyx_L28:; - - /* "trunk/gensim/models/doc2vec_inner.pyx":592 - * saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence2_dm_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/doc2vec_inner.pyx":524 - * return next_random - * - * cdef unsigned long long fast_sentence1_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - - /* function exit code */ - __pyx_L0:; - return __pyx_r; -} - -/* "trunk/gensim/models/doc2vec_inner.pyx":594 - * return next_random - * - * cdef unsigned long long fast_sentence2_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, int *__pyx_v_lbl_codelens, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t *__pyx_v_indexes, __pyx_t_5numpy_uint32_t *__pyx_v_lbl_indexes, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random, int __pyx_v_lbl_length, int __pyx_v_tw, int __pyx_v_tl) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - __pyx_t_5numpy_uint32_t __pyx_v_word_index; - int __pyx_v_d; - int __pyx_v_m; - unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - long __pyx_t_8; - - /* "trunk/gensim/models/doc2vec_inner.pyx":602 - * cdef long long a - * cdef long long row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label - * cdef np.uint32_t target_index, word_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/doc2vec_inner.pyx":607 - * cdef int d, m - * - * word_index = indexes[i] # <<<<<<<<<<<<<< - * - * for a in range(size): - */ - __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":609 - * word_index = indexes[i] - * - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] = 0.0 - * count = 0.0 - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":610 - * - * for a in range(size): - * neu1[a] = 0.0 # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - (__pyx_v_neu1[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":611 - * for a in range(size): - * neu1[a] = 0.0 - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":612 - * neu1[a] = 0.0 - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":613 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L8_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":614 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L5_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":616 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":617 - * else: - * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":618 - * count += ONEF - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L5_continue:; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":619 - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/doc2vec_inner.pyx":620 - * neu1[a] += syn0[indexes[m] * size + a] - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":621 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L12_continue; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":623 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[lbl_indexes[m] * size + a] + * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - - /* "trunk/gensim/models/doc2vec_inner.pyx":624 - * else: - * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[lbl_indexes[m] * size + a] - * if cbow_mean and count > (0.5): - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":625 - * count += ONEF - * for a in range(size): - * neu1[a] += syn0[lbl_indexes[m] * size + a] # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * for a in range(size): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L12_continue:; + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_8); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_9 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_v_next_random = __pyx_t_9; + goto __pyx_L8; } + __pyx_L8:; - /* "trunk/gensim/models/doc2vec_inner.pyx":626 - * for a in range(size): - * neu1[a] += syn0[lbl_indexes[m] * size + a] - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] /= count + /* "trunk/gensim/models/doc2vec_inner.pyx":332 + * + * # convert Python structures to primitive types, so we can release the GIL + * if work is None: # <<<<<<<<<<<<<< + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) */ - __pyx_t_5 = (__pyx_v_cbow_mean != 0); + __pyx_t_4 = (__pyx_v_work == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); if (__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L18_bool_binop_done; + + /* "trunk/gensim/models/doc2vec_inner.pyx":333 + * # convert Python structures to primitive types, so we can release the GIL + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) # <<<<<<<<<<<<<< + * _work = np.PyArray_DATA(work) + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + */ + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_zeros); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_8); + __Pyx_GIVEREF(__pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = PyDict_New(); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = __Pyx_GetModuleGlobalName(__pyx_n_s_REAL); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_t_8, __pyx_n_s_dtype, __pyx_t_10) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_1, __pyx_t_8); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_work, __pyx_t_10); + __pyx_t_10 = 0; + goto __pyx_L9; } - __pyx_t_5 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L18_bool_binop_done:; - if (__pyx_t_4) { + __pyx_L9:; - /* "trunk/gensim/models/doc2vec_inner.pyx":627 - * neu1[a] += syn0[lbl_indexes[m] * size + a] - * if cbow_mean and count > (0.5): - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] /= count - * + /* "trunk/gensim/models/doc2vec_inner.pyx":334 + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) # <<<<<<<<<<<<<< + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + if (!(likely(((__pyx_v_work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 334; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__work = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_work))); - /* "trunk/gensim/models/doc2vec_inner.pyx":628 - * if cbow_mean and count > (0.5): - * for a in range(size): - * neu1[a] /= count # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":335 + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) # <<<<<<<<<<<<<< + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) * - * for a in range(size): */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) / __pyx_v_count); - } - goto __pyx_L17; + __pyx_t_7 = PyObject_Length(__pyx_v_word_vocabs); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 335; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_11 = 10000; + if (((__pyx_t_7 < __pyx_t_11) != 0)) { + __pyx_t_12 = __pyx_t_7; + } else { + __pyx_t_12 = __pyx_t_11; } - __pyx_L17:; + __pyx_v_document_len = ((int)__pyx_t_12); - /* "trunk/gensim/models/doc2vec_inner.pyx":630 - * neu1[a] /= count - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 + /* "trunk/gensim/models/doc2vec_inner.pyx":336 + * _work = np.PyArray_DATA(work) + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) # <<<<<<<<<<<<<< * + * for i in range(document_len): */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_12 = PyObject_Length(__pyx_v_doctag_indexes); if (unlikely(__pyx_t_12 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 336; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_11 = 10000; + if (((__pyx_t_12 < __pyx_t_11) != 0)) { + __pyx_t_7 = __pyx_t_12; + } else { + __pyx_t_7 = __pyx_t_11; + } + __pyx_v_doctag_len = ((int)__pyx_t_7); - /* "trunk/gensim/models/doc2vec_inner.pyx":631 + /* "trunk/gensim/models/doc2vec_inner.pyx":338 + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * - * for d in range(negative+1): + * for i in range(document_len): # <<<<<<<<<<<<<< + * predict_word = word_vocabs[i] + * if predict_word is None: */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":633 - * work[a] = 0.0 + /* "trunk/gensim/models/doc2vec_inner.pyx":339 * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index + * for i in range(document_len): + * predict_word = word_vocabs[i] # <<<<<<<<<<<<<< + * if predict_word is None: + * # shrink document to leave out word */ - __pyx_t_8 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_8; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; + __pyx_t_10 = __Pyx_GetItemInt(__pyx_v_word_vocabs, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_10 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 339; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_10); + __Pyx_XDECREF_SET(__pyx_v_predict_word, __pyx_t_10); + __pyx_t_10 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":634 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF + /* "trunk/gensim/models/doc2vec_inner.pyx":340 + * for i in range(document_len): + * predict_word = word_vocabs[i] + * if predict_word is None: # <<<<<<<<<<<<<< + * # shrink document to leave out word + * document_len = document_len - 1 */ - __pyx_t_4 = ((__pyx_v_d == 0) != 0); + __pyx_t_5 = (__pyx_v_predict_word == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":635 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF + /* "trunk/gensim/models/doc2vec_inner.pyx":342 + * if predict_word is None: + * # shrink document to leave out word + * document_len = document_len - 1 # <<<<<<<<<<<<<< + * continue # leaving j unchanged * else: */ - __pyx_v_target_index = __pyx_v_word_index; + __pyx_v_document_len = (__pyx_v_document_len - 1); - /* "trunk/gensim/models/doc2vec_inner.pyx":636 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":343 + * # shrink document to leave out word + * document_len = document_len - 1 + * continue # leaving j unchanged # <<<<<<<<<<<<<< * else: - * target_index = table[(next_random >> 16) % table_len] + * indexes[i] = predict_word.index */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF; - goto __pyx_L26; + goto __pyx_L10_continue; } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":638 - * label = ONEF + /* "trunk/gensim/models/doc2vec_inner.pyx":345 + * continue # leaving j unchanged * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: + * indexes[i] = predict_word.index # <<<<<<<<<<<<<< + * if hs: + * codelens[i] = len(predict_word.code) */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_predict_word, __pyx_n_s_index); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 345; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_10); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 345; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + (__pyx_v_indexes[__pyx_v_i]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":639 + /* "trunk/gensim/models/doc2vec_inner.pyx":346 * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/doc2vec_inner.pyx":640 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 + * indexes[i] = predict_word.index + * if hs: # <<<<<<<<<<<<<< + * codelens[i] = len(predict_word.code) + * codes[i] = np.PyArray_DATA(predict_word.code) */ - __pyx_t_4 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); + __pyx_t_4 = (__pyx_v_hs != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":641 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L24_continue; - } - - /* "trunk/gensim/models/doc2vec_inner.pyx":642 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size + /* "trunk/gensim/models/doc2vec_inner.pyx":347 + * indexes[i] = predict_word.index + * if hs: + * codelens[i] = len(predict_word.code) # <<<<<<<<<<<<<< + * codes[i] = np.PyArray_DATA(predict_word.code) + * points[i] = np.PyArray_DATA(predict_word.point) */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); - } - __pyx_L26:; + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_predict_word, __pyx_n_s_code); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 347; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_7 = PyObject_Length(__pyx_t_10); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 347; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + (__pyx_v_codelens[__pyx_v_i]) = ((int)__pyx_t_7); - /* "trunk/gensim/models/doc2vec_inner.pyx":644 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): + /* "trunk/gensim/models/doc2vec_inner.pyx":348 + * if hs: + * codelens[i] = len(predict_word.code) + * codes[i] = np.PyArray_DATA(predict_word.code) # <<<<<<<<<<<<<< + * points[i] = np.PyArray_DATA(predict_word.point) + * else: */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_predict_word, __pyx_n_s_code); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 348; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + if (!(likely(((__pyx_t_10) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_10, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 348; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_codes[__pyx_v_i]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_10))); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":645 - * - * row2 = target_index * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] + /* "trunk/gensim/models/doc2vec_inner.pyx":349 + * codelens[i] = len(predict_word.code) + * codes[i] = np.PyArray_DATA(predict_word.code) + * points[i] = np.PyArray_DATA(predict_word.point) # <<<<<<<<<<<<<< + * else: + * codelens[i] = 1 */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_predict_word, __pyx_n_s_point); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 349; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + if (!(likely(((__pyx_t_10) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_10, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 349; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_points[__pyx_v_i]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_10))); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + goto __pyx_L13; + } + /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":646 - * row2 = target_index * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += neu1[a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: + /* "trunk/gensim/models/doc2vec_inner.pyx":351 + * points[i] = np.PyArray_DATA(predict_word.point) + * else: + * codelens[i] = 1 # <<<<<<<<<<<<<< + * result += 1 + * if _train_words: */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + (__pyx_v_codelens[__pyx_v_i]) = 1; + } + __pyx_L13:; - /* "trunk/gensim/models/doc2vec_inner.pyx":647 - * f = 0.0 - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":352 + * else: + * codelens[i] = 1 + * result += 1 # <<<<<<<<<<<<<< + * if _train_words: + * # single randint() call avoids a big thread-synchronization slowdown */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_neu1[__pyx_v_a]) * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); + __pyx_v_result = (__pyx_v_result + 1); } + __pyx_L10_continue:; + } - /* "trunk/gensim/models/doc2vec_inner.pyx":648 - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + /* "trunk/gensim/models/doc2vec_inner.pyx":353 + * codelens[i] = 1 + * result += 1 + * if _train_words: # <<<<<<<<<<<<<< + * # single randint() call avoids a big thread-synchronization slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): */ - __pyx_t_5 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L31_bool_binop_done; - } - __pyx_t_5 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L31_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = (__pyx_v__train_words != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":649 - * f += neu1[a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L24_continue; + /* "trunk/gensim/models/doc2vec_inner.pyx":355 + * if _train_words: + * # single randint() call avoids a big thread-synchronization slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): # <<<<<<<<<<<<<< + * reduced_windows[i] = item + * for i in range(doctag_len): + */ + __pyx_t_2 = 0; + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyInt_From_int(__pyx_v_document_len); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_15 = NULL; + __pyx_t_7 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_15 = PyMethod_GET_SELF(__pyx_t_8); + if (likely(__pyx_t_15)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_15); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_8, function); + __pyx_t_7 = 1; + } } - - /* "trunk/gensim/models/doc2vec_inner.pyx":650 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * for a in range(size): - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/doc2vec_inner.pyx":651 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/doc2vec_inner.pyx":652 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1neg[row2 + a] - * if tw: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/doc2vec_inner.pyx":653 - * g = (label - f) * alpha - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * if tw: - * for a in range(size): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); + __pyx_t_16 = PyTuple_New(3+__pyx_t_7); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_16); + if (__pyx_t_15) { + PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_15); __Pyx_GIVEREF(__pyx_t_15); __pyx_t_15 = NULL; } + __Pyx_INCREF(__pyx_int_0); + PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_7, __pyx_int_0); + __Pyx_GIVEREF(__pyx_int_0); + PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_7, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_16, 2+__pyx_t_7, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_6); + __pyx_t_1 = 0; + __pyx_t_6 = 0; + __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (likely(PyList_CheckExact(__pyx_t_10)) || PyTuple_CheckExact(__pyx_t_10)) { + __pyx_t_8 = __pyx_t_10; __Pyx_INCREF(__pyx_t_8); __pyx_t_7 = 0; + __pyx_t_17 = NULL; + } else { + __pyx_t_7 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_10); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_17 = Py_TYPE(__pyx_t_8)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + for (;;) { + if (likely(!__pyx_t_17)) { + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_10 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_7); __Pyx_INCREF(__pyx_t_10); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #else + __pyx_t_10 = PySequence_ITEM(__pyx_t_8, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + } else { + if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_10 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_7); __Pyx_INCREF(__pyx_t_10); __pyx_t_7++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #else + __pyx_t_10 = PySequence_ITEM(__pyx_t_8, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + } + } else { + __pyx_t_10 = __pyx_t_17(__pyx_t_8); + if (unlikely(!__pyx_t_10)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + break; + } + __Pyx_GOTREF(__pyx_t_10); + } + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_10); + __pyx_t_10 = 0; + __pyx_v_i = __pyx_t_2; + __pyx_t_2 = (__pyx_t_2 + 1); + + /* "trunk/gensim/models/doc2vec_inner.pyx":356 + * # single randint() call avoids a big thread-synchronization slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): + * reduced_windows[i] = item # <<<<<<<<<<<<<< + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] + */ + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 356; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_14; + + /* "trunk/gensim/models/doc2vec_inner.pyx":355 + * if _train_words: + * # single randint() call avoids a big thread-synchronization slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): # <<<<<<<<<<<<<< + * reduced_windows[i] = item + * for i in range(doctag_len): + */ + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L14; + } + __pyx_L14:; - /* "trunk/gensim/models/doc2vec_inner.pyx":654 - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - * if tw: # <<<<<<<<<<<<<< - * for a in range(size): - * syn1neg[row2 + a] += g * neu1[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":357 + * for i, item in enumerate(np.random.randint(0, window, document_len)): + * reduced_windows[i] = item + * for i in range(doctag_len): # <<<<<<<<<<<<<< + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 */ - __pyx_t_4 = (__pyx_v_tw != 0); - if (__pyx_t_4) { + __pyx_t_2 = __pyx_v_doctag_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":655 - * work[a] += g * syn1neg[row2 + a] - * if tw: - * for a in range(size): # <<<<<<<<<<<<<< - * syn1neg[row2 + a] += g * neu1[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":358 + * reduced_windows[i] = item + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] # <<<<<<<<<<<<<< + * result += 1 * */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_doctag_indexes, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_8 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 358; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_8); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 358; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + (__pyx_v__doctag_indexes[__pyx_v_i]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":656 - * if tw: - * for a in range(size): - * syn1neg[row2 + a] += g * neu1[a] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":359 + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 # <<<<<<<<<<<<<< * - * if tw: + * # release GIL & train on the document */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1neg[__pyx_t_7]) = ((__pyx_v_syn1neg[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_neu1[__pyx_v_a]))); - } - goto __pyx_L35; - } - __pyx_L35:; - __pyx_L24_continue:; + __pyx_v_result = (__pyx_v_result + 1); } - /* "trunk/gensim/models/doc2vec_inner.pyx":658 - * syn1neg[row2 + a] += g * neu1[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":362 * - * if tw: # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: + * # release GIL & train on the document + * with nogil: # <<<<<<<<<<<<<< + * for i in range(document_len): + * if codelens[i] == 0: */ - __pyx_t_4 = (__pyx_v_tw != 0); - if (__pyx_t_4) { + { + #ifdef WITH_THREAD + PyThreadState *_save; + Py_UNBLOCK_THREADS + #endif + /*try:*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":659 - * - * if tw: - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":363 + * # release GIL & train on the document + * with nogil: + * for i in range(document_len): # <<<<<<<<<<<<<< + * if codelens[i] == 0: * continue */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":660 - * if tw: - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":364 + * with nogil: + * for i in range(document_len): + * if codelens[i] == 0: # <<<<<<<<<<<<<< * continue - * else: + * if _train_words: # simultaneous skip-gram wordvec-training */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L42_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L42_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_i]) == 0) != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":661 - * for m in range(j, k): - * if m == i or codelens[m] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":365 + * for i in range(document_len): + * if codelens[i] == 0: * continue # <<<<<<<<<<<<<< - * else: - * for a in range(size): + * if _train_words: # simultaneous skip-gram wordvec-training + * j = i - window + reduced_windows[i] */ - goto __pyx_L39_continue; - } - /*else*/ { + goto __pyx_L22_continue; + } - /* "trunk/gensim/models/doc2vec_inner.pyx":663 + /* "trunk/gensim/models/doc2vec_inner.pyx":366 + * if codelens[i] == 0: * continue - * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[indexes[m] * size + a] += work[a] - * if tl: + * if _train_words: # simultaneous skip-gram wordvec-training # <<<<<<<<<<<<<< + * j = i - window + reduced_windows[i] + * if j < 0: */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_4 = (__pyx_v__train_words != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":664 - * else: - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< - * if tl: - * for m in range(lbl_length): + /* "trunk/gensim/models/doc2vec_inner.pyx":367 + * continue + * if _train_words: # simultaneous skip-gram wordvec-training + * j = i - window + reduced_windows[i] # <<<<<<<<<<<<<< + * if j < 0: + * j = 0 */ - __pyx_t_7 = (((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - } - __pyx_L39_continue:; - } - goto __pyx_L38; - } - __pyx_L38:; + __pyx_v_j = ((__pyx_v_i - __pyx_v_window) + (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/doc2vec_inner.pyx":665 - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] - * if tl: # <<<<<<<<<<<<<< - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":368 + * if _train_words: # simultaneous skip-gram wordvec-training + * j = i - window + reduced_windows[i] + * if j < 0: # <<<<<<<<<<<<<< + * j = 0 + * k = i + window + 1 - reduced_windows[i] */ - __pyx_t_4 = (__pyx_v_tl != 0); - if (__pyx_t_4) { + __pyx_t_4 = ((__pyx_v_j < 0) != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":666 - * syn0[indexes[m] * size + a] += work[a] - * if tl: - * for m in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[m] == 0: - * continue + /* "trunk/gensim/models/doc2vec_inner.pyx":369 + * j = i - window + reduced_windows[i] + * if j < 0: + * j = 0 # <<<<<<<<<<<<<< + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: */ - __pyx_t_1 = __pyx_v_lbl_length; - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; + __pyx_v_j = 0; + goto __pyx_L26; + } + __pyx_L26:; - /* "trunk/gensim/models/doc2vec_inner.pyx":667 - * if tl: - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: + /* "trunk/gensim/models/doc2vec_inner.pyx":370 + * if j < 0: + * j = 0 + * k = i + window + 1 - reduced_windows[i] # <<<<<<<<<<<<<< + * if k > document_len: + * k = document_len + */ + __pyx_v_k = (((__pyx_v_i + __pyx_v_window) + 1) - (__pyx_v_reduced_windows[__pyx_v_i])); + + /* "trunk/gensim/models/doc2vec_inner.pyx":371 + * j = 0 + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: # <<<<<<<<<<<<<< + * k = document_len + * for j in range(j, k): + */ + __pyx_t_4 = ((__pyx_v_k > __pyx_v_document_len) != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":372 + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: + * k = document_len # <<<<<<<<<<<<<< + * for j in range(j, k): + * if j == i or codelens[j] == 0: + */ + __pyx_v_k = __pyx_v_document_len; + goto __pyx_L27; + } + __pyx_L27:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":373 + * if k > document_len: + * k = document_len + * for j in range(j, k): # <<<<<<<<<<<<<< + * if j == i or codelens[j] == 0: + * continue + */ + __pyx_t_11 = __pyx_v_k; + for (__pyx_t_18 = __pyx_v_j; __pyx_t_18 < __pyx_t_11; __pyx_t_18+=1) { + __pyx_v_j = __pyx_t_18; + + /* "trunk/gensim/models/doc2vec_inner.pyx":374 + * k = document_len + * for j in range(j, k): + * if j == i or codelens[j] == 0: # <<<<<<<<<<<<<< + * continue + * if hs: + */ + __pyx_t_5 = ((__pyx_v_j == __pyx_v_i) != 0); + if (!__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L31_bool_binop_done; + } + __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_j]) == 0) != 0); + __pyx_t_4 = __pyx_t_5; + __pyx_L31_bool_binop_done:; + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":375 + * for j in range(j, k): + * if j == i or codelens[j] == 0: + * continue # <<<<<<<<<<<<<< + * if hs: + * # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + */ + goto __pyx_L28_continue; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":376 + * if j == i or codelens[j] == 0: + * continue + * if hs: # <<<<<<<<<<<<<< + * # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _word_vectors, syn1, size, indexes[j], + */ + __pyx_t_4 = (__pyx_v_hs != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":378 + * if hs: + * # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _word_vectors, syn1, size, indexes[j], # <<<<<<<<<<<<<< + * _alpha, _work, _learn_words, _learn_hidden, _word_locks) + * if negative: + */ + __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v__word_vectors, __pyx_v_syn1, __pyx_v_size, (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v__work, __pyx_v__learn_words, __pyx_v__learn_hidden, __pyx_v__word_locks); + goto __pyx_L33; + } + __pyx_L33:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":380 + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _word_vectors, syn1, size, indexes[j], + * _alpha, _work, _learn_words, _learn_hidden, _word_locks) + * if negative: # <<<<<<<<<<<<<< + * # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + * next_random = fast_document_dbow_neg(negative, table, table_len, _word_vectors, syn1neg, size, + */ + __pyx_t_4 = (__pyx_v_negative != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":382 + * if negative: + * # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + * next_random = fast_document_dbow_neg(negative, table, table_len, _word_vectors, syn1neg, size, # <<<<<<<<<<<<<< + * indexes[i], indexes[j], _alpha, _work, next_random, + * _learn_words, _learn_hidden, _word_locks) + */ + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v__word_vectors, __pyx_v_syn1neg, __pyx_v_size, (__pyx_v_indexes[__pyx_v_i]), (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v__work, __pyx_v_next_random, __pyx_v__learn_words, __pyx_v__learn_hidden, __pyx_v__word_locks); + goto __pyx_L34; + } + __pyx_L34:; + __pyx_L28_continue:; + } + goto __pyx_L25; + } + __pyx_L25:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":387 + * + * # docvec-training + * for j in range(doctag_len): # <<<<<<<<<<<<<< + * if hs: + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _doctag_vectors, syn1, size, _doctag_indexes[j], + */ + __pyx_t_18 = __pyx_v_doctag_len; + for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { + __pyx_v_j = __pyx_t_19; + + /* "trunk/gensim/models/doc2vec_inner.pyx":388 + * # docvec-training + * for j in range(doctag_len): + * if hs: # <<<<<<<<<<<<<< + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _doctag_vectors, syn1, size, _doctag_indexes[j], + * _alpha, _work, _learn_doctags, _learn_hidden, _doctag_locks) + */ + __pyx_t_4 = (__pyx_v_hs != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":389 + * for j in range(doctag_len): + * if hs: + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _doctag_vectors, syn1, size, _doctag_indexes[j], # <<<<<<<<<<<<<< + * _alpha, _work, _learn_doctags, _learn_hidden, _doctag_locks) + * if negative: */ - __pyx_t_4 = (((__pyx_v_lbl_codelens[__pyx_v_m]) == 0) != 0); - if (__pyx_t_4) { + __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v__doctag_vectors, __pyx_v_syn1, __pyx_v_size, (__pyx_v__doctag_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v__work, __pyx_v__learn_doctags, __pyx_v__learn_hidden, __pyx_v__doctag_locks); + goto __pyx_L37; + } + __pyx_L37:; - /* "trunk/gensim/models/doc2vec_inner.pyx":668 - * for m in range(lbl_length): - * if lbl_codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * for a in range(size): + /* "trunk/gensim/models/doc2vec_inner.pyx":391 + * fast_document_dbow_hs(points[i], codes[i], codelens[i], _doctag_vectors, syn1, size, _doctag_indexes[j], + * _alpha, _work, _learn_doctags, _learn_hidden, _doctag_locks) + * if negative: # <<<<<<<<<<<<<< + * next_random = fast_document_dbow_neg(negative, table, table_len, _doctag_vectors, syn1neg, size, + * indexes[i], _doctag_indexes[j], _alpha, _work, next_random, */ - goto __pyx_L47_continue; - } - /*else*/ { + __pyx_t_4 = (__pyx_v_negative != 0); + if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":670 - * continue - * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[lbl_indexes[m] * size + a] += work[a] - * + /* "trunk/gensim/models/doc2vec_inner.pyx":392 + * _alpha, _work, _learn_doctags, _learn_hidden, _doctag_locks) + * if negative: + * next_random = fast_document_dbow_neg(negative, table, table_len, _doctag_vectors, syn1neg, size, # <<<<<<<<<<<<<< + * indexes[i], _doctag_indexes[j], _alpha, _work, next_random, + * _learn_doctags, _learn_hidden, _doctag_locks) */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dbow_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v__doctag_vectors, __pyx_v_syn1neg, __pyx_v_size, (__pyx_v_indexes[__pyx_v_i]), (__pyx_v__doctag_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v__work, __pyx_v_next_random, __pyx_v__learn_doctags, __pyx_v__learn_hidden, __pyx_v__doctag_locks); + goto __pyx_L38; + } + __pyx_L38:; + } + __pyx_L22_continue:; + } + } - /* "trunk/gensim/models/doc2vec_inner.pyx":671 - * else: - * for a in range(size): - * syn0[lbl_indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":362 * - * return next_random + * # release GIL & train on the document + * with nogil: # <<<<<<<<<<<<<< + * for i in range(document_len): + * if codelens[i] == 0: */ - __pyx_t_7 = (((__pyx_v_lbl_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); + /*finally:*/ { + /*normal exit:*/{ + #ifdef WITH_THREAD + Py_BLOCK_THREADS + #endif + goto __pyx_L21; } + __pyx_L21:; } - __pyx_L47_continue:; - } - goto __pyx_L46; } - __pyx_L46:; - /* "trunk/gensim/models/doc2vec_inner.pyx":673 - * syn0[lbl_indexes[m] * size + a] += work[a] + /* "trunk/gensim/models/doc2vec_inner.pyx":396 + * _learn_doctags, _learn_hidden, _doctag_locks) + * + * return result # <<<<<<<<<<<<<< * - * return next_random # <<<<<<<<<<<<<< * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): */ - __pyx_r = __pyx_v_next_random; + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 396; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":594 - * return next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":268 + * * - * cdef unsigned long long fast_sentence2_dm_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, # <<<<<<<<<<<<<< + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_XDECREF(__pyx_t_16); + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dbow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; __pyx_L0:; + __Pyx_XDECREF(__pyx_v_predict_word); + __Pyx_XDECREF(__pyx_v_item); + __Pyx_XDECREF(__pyx_v_work); + __Pyx_XDECREF(__pyx_v_word_vectors); + __Pyx_XDECREF(__pyx_v_word_locks); + __Pyx_XDECREF(__pyx_v_doctag_vectors); + __Pyx_XDECREF(__pyx_v_doctag_locks); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":675 - * return next_random +/* "trunk/gensim/models/doc2vec_inner.pyx":399 * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ /* Python wrapper */ -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence_dbow(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence_dbow = {"train_sentence_dbow", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence_dbow, METH_VARARGS|METH_KEYWORDS, 0}; -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence_dbow(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_document_dm(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_3train_document_dm = {"train_document_dm", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_document_dm, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_document_dm(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_model = 0; - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_lbls = 0; + PyObject *__pyx_v_word_vocabs = 0; + PyObject *__pyx_v_doctag_indexes = 0; PyObject *__pyx_v_alpha = 0; - PyObject *__pyx_v__work = 0; - PyObject *__pyx_v_train_words = 0; - PyObject *__pyx_v_train_lbls = 0; + PyObject *__pyx_v_work = 0; + PyObject *__pyx_v_neu1 = 0; + PyObject *__pyx_v_learn_doctags = 0; + PyObject *__pyx_v_learn_words = 0; + PyObject *__pyx_v_learn_hidden = 0; + PyObject *__pyx_v_word_vectors = 0; + PyObject *__pyx_v_word_locks = 0; + PyObject *__pyx_v_doctag_vectors = 0; + PyObject *__pyx_v_doctag_locks = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("train_sentence_dbow (wrapper)", 0); + __Pyx_RefNannySetupContext("train_document_dm (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_model,&__pyx_n_s_sentence,&__pyx_n_s_lbls,&__pyx_n_s_alpha,&__pyx_n_s_work,&__pyx_n_s_train_words,&__pyx_n_s_train_lbls,0}; - PyObject* values[7] = {0,0,0,0,0,0,0}; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_model,&__pyx_n_s_word_vocabs,&__pyx_n_s_doctag_indexes,&__pyx_n_s_alpha,&__pyx_n_s_work,&__pyx_n_s_neu1,&__pyx_n_s_learn_doctags,&__pyx_n_s_learn_words,&__pyx_n_s_learn_hidden,&__pyx_n_s_word_vectors,&__pyx_n_s_word_locks,&__pyx_n_s_doctag_vectors,&__pyx_n_s_doctag_locks,0}; + PyObject* values[13] = {0,0,0,0,0,0,0,0,0,0,0,0,0}; + values[4] = ((PyObject *)Py_None); + values[5] = ((PyObject *)Py_None); + + /* "trunk/gensim/models/doc2vec_inner.pyx":400 + * + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + * learn_doctags=True, learn_words=True, learn_hidden=True, # <<<<<<<<<<<<<< + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + * cdef int hs = model.hs + */ + values[6] = ((PyObject *)Py_True); + values[7] = ((PyObject *)Py_True); + values[8] = ((PyObject *)Py_True); + + /* "trunk/gensim/models/doc2vec_inner.pyx":401 + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): # <<<<<<<<<<<<<< + * cdef int hs = model.hs + * cdef int negative = model.negative + */ + values[9] = ((PyObject *)Py_None); + values[10] = ((PyObject *)Py_None); + values[11] = ((PyObject *)Py_None); + values[12] = ((PyObject *)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); @@ -6241,92 +4278,153 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_model)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vocabs)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm", 0, 4, 13, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_lbls)) != 0)) kw_args--; + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_indexes)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm", 0, 4, 13, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_alpha)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm", 0, 4, 13, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 4: - if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work); + if (value) { values[4] = value; kw_args--; } } case 5: - if (likely((values[5] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_train_words)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 5); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_neu1); + if (value) { values[5] = value; kw_args--; } } case 6: - if (likely((values[6] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_train_lbls)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, 6); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_doctags); + if (value) { values[6] = value; kw_args--; } + } + case 7: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_words); + if (value) { values[7] = value; kw_args--; } + } + case 8: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_hidden); + if (value) { values[8] = value; kw_args--; } + } + case 9: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vectors); + if (value) { values[9] = value; kw_args--; } + } + case 10: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_locks); + if (value) { values[10] = value; kw_args--; } + } + case 11: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_vectors); + if (value) { values[11] = value; kw_args--; } + } + case 12: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_locks); + if (value) { values[12] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_dbow") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_document_dm") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } - } else if (PyTuple_GET_SIZE(__pyx_args) != 7) { - goto __pyx_L5_argtuple_error; } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } } __pyx_v_model = values[0]; - __pyx_v_sentence = values[1]; - __pyx_v_lbls = values[2]; + __pyx_v_word_vocabs = values[1]; + __pyx_v_doctag_indexes = values[2]; __pyx_v_alpha = values[3]; - __pyx_v__work = values[4]; - __pyx_v_train_words = values[5]; - __pyx_v_train_lbls = values[6]; + __pyx_v_work = values[4]; + __pyx_v_neu1 = values[5]; + __pyx_v_learn_doctags = values[6]; + __pyx_v_learn_words = values[7]; + __pyx_v_learn_hidden = values[8]; + __pyx_v_word_vectors = values[9]; + __pyx_v_word_locks = values[10]; + __pyx_v_doctag_vectors = values[11]; + __pyx_v_doctag_locks = values[12]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("train_sentence_dbow", 1, 7, 7, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm", 0, 4, 13, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; - __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_sentence_dbow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dm", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_dbow(__pyx_self, __pyx_v_model, __pyx_v_sentence, __pyx_v_lbls, __pyx_v_alpha, __pyx_v__work, __pyx_v_train_words, __pyx_v_train_lbls); + __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_document_dm(__pyx_self, __pyx_v_model, __pyx_v_word_vocabs, __pyx_v_doctag_indexes, __pyx_v_alpha, __pyx_v_work, __pyx_v_neu1, __pyx_v_learn_doctags, __pyx_v_learn_words, __pyx_v_learn_hidden, __pyx_v_word_vectors, __pyx_v_word_locks, __pyx_v_doctag_vectors, __pyx_v_doctag_locks); + + /* "trunk/gensim/models/doc2vec_inner.pyx":399 + * + * + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + */ /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_dbow(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_sentence, PyObject *__pyx_v_lbls, PyObject *__pyx_v_alpha, PyObject *__pyx_v__work, PyObject *__pyx_v_train_words, PyObject *__pyx_v_train_lbls) { +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_document_dm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_neu1, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks) { int __pyx_v_hs; int __pyx_v_negative; - int __pyx_v_tw; - int __pyx_v_tl; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work; + int __pyx_v__learn_doctags; + int __pyx_v__learn_words; + int __pyx_v__learn_hidden; + int __pyx_v_cbow_mean; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_count; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v_inv_count; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__work; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__neu1; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v__alpha; int __pyx_v_size; int __pyx_v_codelens[10000]; - int __pyx_v_lbl_codelens[10000]; __pyx_t_5numpy_uint32_t __pyx_v_indexes[10000]; - __pyx_t_5numpy_uint32_t __pyx_v_lbl_indexes[10000]; + __pyx_t_5numpy_uint32_t __pyx_v__doctag_indexes[10000]; __pyx_t_5numpy_uint32_t __pyx_v_reduced_windows[10000]; - int __pyx_v_sentence_len; - int __pyx_v_lbl_length; + int __pyx_v_document_len; + int __pyx_v_doctag_len; int __pyx_v_window; int __pyx_v_i; int __pyx_v_j; + int __pyx_v_k; + int __pyx_v_m; long __pyx_v_result; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1; __pyx_t_5numpy_uint32_t *__pyx_v_points[10000]; @@ -6343,16 +4441,16 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_ int __pyx_t_2; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_t_3; int __pyx_t_4; - Py_ssize_t __pyx_t_5; + int __pyx_t_5; PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - unsigned PY_LONG_LONG __pyx_t_8; - long __pyx_t_9; - Py_ssize_t __pyx_t_10; - int __pyx_t_11; - int __pyx_t_12; - __pyx_t_5numpy_uint32_t __pyx_t_13; - PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + unsigned PY_LONG_LONG __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + long __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_t_13; + __pyx_t_5numpy_uint32_t __pyx_t_14; PyObject *__pyx_t_15 = NULL; PyObject *__pyx_t_16 = NULL; PyObject *(*__pyx_t_17)(PyObject *); @@ -6361,114 +4459,289 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_ int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("train_sentence_dbow", 0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":676 - * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): + __Pyx_RefNannySetupContext("train_document_dm", 0); + __Pyx_INCREF(__pyx_v_work); + __Pyx_INCREF(__pyx_v_neu1); + __Pyx_INCREF(__pyx_v_word_vectors); + __Pyx_INCREF(__pyx_v_word_locks); + __Pyx_INCREF(__pyx_v_doctag_vectors); + __Pyx_INCREF(__pyx_v_doctag_locks); + + /* "trunk/gensim/models/doc2vec_inner.pyx":402 + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): * cdef int hs = model.hs # <<<<<<<<<<<<<< * cdef int negative = model.negative - * cdef int tw = train_words + * cdef int _learn_doctags = learn_doctags */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 676; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 402; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 676; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 402; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_hs = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":677 - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): + /* "trunk/gensim/models/doc2vec_inner.pyx":403 + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): * cdef int hs = model.hs * cdef int negative = model.negative # <<<<<<<<<<<<<< - * cdef int tw = train_words - * cdef int tl = train_lbls + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 677; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 403; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 677; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 403; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_negative = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":678 + /* "trunk/gensim/models/doc2vec_inner.pyx":404 * cdef int hs = model.hs * cdef int negative = model.negative - * cdef int tw = train_words # <<<<<<<<<<<<<< - * cdef int tl = train_lbls - * + * cdef int _learn_doctags = learn_doctags # <<<<<<<<<<<<<< + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_train_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 678; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_tw = __pyx_t_2; + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_doctags); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 404; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_doctags = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":679 + /* "trunk/gensim/models/doc2vec_inner.pyx":405 * cdef int negative = model.negative - * cdef int tw = train_words - * cdef int tl = train_lbls # <<<<<<<<<<<<<< - * - * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words # <<<<<<<<<<<<<< + * cdef int _learn_hidden = learn_hidden + * cdef int cbow_mean = model.cbow_mean */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_train_lbls); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 679; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_tl = __pyx_t_2; + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 405; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_words = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":681 - * cdef int tl = train_lbls + /* "trunk/gensim/models/doc2vec_inner.pyx":406 + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden # <<<<<<<<<<<<<< + * cdef int cbow_mean = model.cbow_mean + * cdef REAL_t count, inv_count = 1.0 + */ + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_hidden); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 406; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_hidden = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":407 + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden + * cdef int cbow_mean = model.cbow_mean # <<<<<<<<<<<<<< + * cdef REAL_t count, inv_count = 1.0 * - * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) # <<<<<<<<<<<<<< - * cdef REAL_t *work - * cdef REAL_t _alpha = alpha */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 681; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_cbow_mean); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 407; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 681; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_syn0 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 407; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_cbow_mean = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":408 + * cdef int _learn_hidden = learn_hidden + * cdef int cbow_mean = model.cbow_mean + * cdef REAL_t count, inv_count = 1.0 # <<<<<<<<<<<<<< + * + * cdef REAL_t *_word_vectors + */ + __pyx_v_inv_count = 1.0; - /* "trunk/gensim/models/doc2vec_inner.pyx":683 - * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) - * cdef REAL_t *work + /* "trunk/gensim/models/doc2vec_inner.pyx":416 + * cdef REAL_t *_work + * cdef REAL_t *_neu1 * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< * cdef int size = model.layer1_size * */ - __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 683; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 416; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v__alpha = __pyx_t_3; - /* "trunk/gensim/models/doc2vec_inner.pyx":684 - * cdef REAL_t *work + /* "trunk/gensim/models/doc2vec_inner.pyx":417 + * cdef REAL_t *_neu1 * cdef REAL_t _alpha = alpha * cdef int size = model.layer1_size # <<<<<<<<<<<<<< * - * cdef int codelens[MAX_SENTENCE_LEN] + * cdef int codelens[MAX_DOCUMENT_LEN] */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 684; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 417; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 684; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 417; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_size = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":693 - * cdef int sentence_len - * cdef int lbl_length + /* "trunk/gensim/models/doc2vec_inner.pyx":425 + * cdef int document_len + * cdef int doctag_len * cdef int window = model.window # <<<<<<<<<<<<<< * - * cdef int i, j + * cdef int i, j, k, m */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 693; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 425; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 693; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 425; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_window = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":696 + /* "trunk/gensim/models/doc2vec_inner.pyx":428 * - * cdef int i, j + * cdef int i, j, k, m * cdef long result = 0 # <<<<<<<<<<<<<< * * # For hierarchical softmax */ __pyx_v_result = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":709 - * cdef unsigned long long next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":442 + * + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: # <<<<<<<<<<<<<< + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + */ + __pyx_t_4 = (__pyx_v_word_vectors == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":443 + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: + * word_vectors = model.syn0 # <<<<<<<<<<<<<< + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 443; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_word_vectors, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L3; + } + __pyx_L3:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":444 + * if word_vectors is None: + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) # <<<<<<<<<<<<<< + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 + */ + if (!(likely(((__pyx_v_word_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 444; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_vectors))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":445 + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: # <<<<<<<<<<<<<< + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + */ + __pyx_t_5 = (__pyx_v_doctag_vectors == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":446 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 # <<<<<<<<<<<<<< + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 446; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_doctag_syn0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 446; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_vectors, __pyx_t_6); + __pyx_t_6 = 0; + goto __pyx_L4; + } + __pyx_L4:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":447 + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) # <<<<<<<<<<<<<< + * if word_locks is None: + * word_locks = model.syn0_lockf + */ + if (!(likely(((__pyx_v_doctag_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 447; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_vectors))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":448 + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: # <<<<<<<<<<<<<< + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + */ + __pyx_t_4 = (__pyx_v_word_locks == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":449 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: + * word_locks = model.syn0_lockf # <<<<<<<<<<<<<< + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: + */ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0_lockf); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 449; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF_SET(__pyx_v_word_locks, __pyx_t_6); + __pyx_t_6 = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":450 + * if word_locks is None: + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) # <<<<<<<<<<<<<< + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + */ + if (!(likely(((__pyx_v_word_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 450; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_locks))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":451 + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: # <<<<<<<<<<<<<< + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) + */ + __pyx_t_5 = (__pyx_v_doctag_locks == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":452 + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf # <<<<<<<<<<<<<< + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) + * + */ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 452; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_doctag_syn0_lockf); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 452; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_locks, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":453 + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) # <<<<<<<<<<<<<< + * + * if hs: + */ + if (!(likely(((__pyx_v_doctag_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 453; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_locks))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":455 + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) * * if hs: # <<<<<<<<<<<<<< * syn1 = (np.PyArray_DATA(model.syn1)) @@ -6477,23 +4750,23 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_ __pyx_t_4 = (__pyx_v_hs != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":710 + /* "trunk/gensim/models/doc2vec_inner.pyx":456 * * if hs: * syn1 = (np.PyArray_DATA(model.syn1)) # <<<<<<<<<<<<<< * * if negative: */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 710; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 456; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 710; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 456; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - goto __pyx_L3; + goto __pyx_L7; } - __pyx_L3:; + __pyx_L7:; - /* "trunk/gensim/models/doc2vec_inner.pyx":712 + /* "trunk/gensim/models/doc2vec_inner.pyx":458 * syn1 = (np.PyArray_DATA(model.syn1)) * * if negative: # <<<<<<<<<<<<<< @@ -6503,491 +4776,530 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_ __pyx_t_4 = (__pyx_v_negative != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":713 + /* "trunk/gensim/models/doc2vec_inner.pyx":459 * * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) # <<<<<<<<<<<<<< * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 713; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 459; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 713; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 459; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1neg = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":714 + /* "trunk/gensim/models/doc2vec_inner.pyx":460 * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) # <<<<<<<<<<<<<< * table_len = len(model.table) - * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) + * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 714; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 460; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 714; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 460; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_table = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":715 + /* "trunk/gensim/models/doc2vec_inner.pyx":461 * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) # <<<<<<<<<<<<<< - * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) + * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 715; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 461; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 715; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 461; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_table_len = __pyx_t_5; + __pyx_v_table_len = __pyx_t_7; - /* "trunk/gensim/models/doc2vec_inner.pyx":716 + /* "trunk/gensim/models/doc2vec_inner.pyx":462 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) - * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< + * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_8); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_9 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_v_next_random = __pyx_t_8; - goto __pyx_L4; + __pyx_v_next_random = __pyx_t_9; + goto __pyx_L8; } - __pyx_L4:; + __pyx_L8:; - /* "trunk/gensim/models/doc2vec_inner.pyx":719 + /* "trunk/gensim/models/doc2vec_inner.pyx":465 * * # convert Python structures to primitive types, so we can release the GIL - * work = np.PyArray_DATA(_work) # <<<<<<<<<<<<<< - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) + * if work is None: # <<<<<<<<<<<<<< + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) */ - if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 719; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_work = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__work))); + __pyx_t_4 = (__pyx_v_work == Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":720 + /* "trunk/gensim/models/doc2vec_inner.pyx":466 * # convert Python structures to primitive types, so we can release the GIL - * work = np.PyArray_DATA(_work) - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) # <<<<<<<<<<<<<< - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) # <<<<<<<<<<<<<< + * _work = np.PyArray_DATA(work) + * if neu1 is None: + */ + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_zeros); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_8); + __Pyx_GIVEREF(__pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = PyDict_New(); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = __Pyx_GetModuleGlobalName(__pyx_n_s_REAL); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_t_8, __pyx_n_s_dtype, __pyx_t_10) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_1, __pyx_t_8); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 466; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_work, __pyx_t_10); + __pyx_t_10 = 0; + goto __pyx_L9; + } + __pyx_L9:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":467 + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) # <<<<<<<<<<<<<< + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) + */ + if (!(likely(((__pyx_v_work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 467; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__work = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_work))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":468 + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) + * if neu1 is None: # <<<<<<<<<<<<<< + * neu1 = zeros(model.layer1_size, dtype=REAL) + * _neu1 = np.PyArray_DATA(neu1) + */ + __pyx_t_5 = (__pyx_v_neu1 == Py_None); + __pyx_t_4 = (__pyx_t_5 != 0); + if (__pyx_t_4) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":469 + * _work = np.PyArray_DATA(work) + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) # <<<<<<<<<<<<<< + * _neu1 = np.PyArray_DATA(neu1) * */ - __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = 10000; - if (((__pyx_t_5 < __pyx_t_9) != 0)) { - __pyx_t_10 = __pyx_t_5; - } else { - __pyx_t_10 = __pyx_t_9; + __pyx_t_10 = __Pyx_GetModuleGlobalName(__pyx_n_s_zeros); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_8); + __Pyx_GIVEREF(__pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_8 = PyDict_New(); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_REAL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_8, __pyx_n_s_dtype, __pyx_t_6) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_10, __pyx_t_1, __pyx_t_8); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 469; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_neu1, __pyx_t_6); + __pyx_t_6 = 0; + goto __pyx_L10; } - __pyx_v_sentence_len = ((int)__pyx_t_10); + __pyx_L10:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":470 + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) + * _neu1 = np.PyArray_DATA(neu1) # <<<<<<<<<<<<<< + * + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + */ + if (!(likely(((__pyx_v_neu1) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_neu1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 470; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__neu1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_neu1))); - /* "trunk/gensim/models/doc2vec_inner.pyx":721 - * work = np.PyArray_DATA(_work) - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":472 + * _neu1 = np.PyArray_DATA(neu1) * - * for i in range(sentence_len): + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) # <<<<<<<<<<<<<< + * j = 0 + * for i in range(document_len): */ - __pyx_t_10 = PyObject_Length(__pyx_v_lbls); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 721; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = 10000; - if (((__pyx_t_10 < __pyx_t_9) != 0)) { - __pyx_t_5 = __pyx_t_10; + __pyx_t_7 = PyObject_Length(__pyx_v_word_vocabs); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 472; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_11 = 10000; + if (((__pyx_t_7 < __pyx_t_11) != 0)) { + __pyx_t_12 = __pyx_t_7; } else { - __pyx_t_5 = __pyx_t_9; + __pyx_t_12 = __pyx_t_11; } - __pyx_v_lbl_length = ((int)__pyx_t_5); + __pyx_v_document_len = ((int)__pyx_t_12); - /* "trunk/gensim/models/doc2vec_inner.pyx":723 - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) + /* "trunk/gensim/models/doc2vec_inner.pyx":473 * - * for i in range(sentence_len): # <<<<<<<<<<<<<< - * word = sentence[i] + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * j = 0 # <<<<<<<<<<<<<< + * for i in range(document_len): + * word = word_vocabs[i] + */ + __pyx_v_j = 0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":474 + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * j = 0 + * for i in range(document_len): # <<<<<<<<<<<<<< + * word = word_vocabs[i] * if word is None: */ - __pyx_t_2 = __pyx_v_sentence_len; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_i = __pyx_t_11; + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":724 - * - * for i in range(sentence_len): - * word = sentence[i] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":475 + * j = 0 + * for i in range(document_len): + * word = word_vocabs[i] # <<<<<<<<<<<<<< * if word is None: - * codelens[i] = 0 + * # shrink document to leave out word */ - __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 724; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_word_vocabs, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 475; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __Pyx_GOTREF(__pyx_t_6); __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":725 - * for i in range(sentence_len): - * word = sentence[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":476 + * for i in range(document_len): + * word = word_vocabs[i] * if word is None: # <<<<<<<<<<<<<< - * codelens[i] = 0 - * else: + * # shrink document to leave out word + * document_len = document_len - 1 */ __pyx_t_4 = (__pyx_v_word == Py_None); - __pyx_t_12 = (__pyx_t_4 != 0); - if (__pyx_t_12) { + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":726 - * word = sentence[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":478 * if word is None: - * codelens[i] = 0 # <<<<<<<<<<<<<< + * # shrink document to leave out word + * document_len = document_len - 1 # <<<<<<<<<<<<<< + * continue # leaving j unchanged + * else: + */ + __pyx_v_document_len = (__pyx_v_document_len - 1); + + /* "trunk/gensim/models/doc2vec_inner.pyx":479 + * # shrink document to leave out word + * document_len = document_len - 1 + * continue # leaving j unchanged # <<<<<<<<<<<<<< * else: - * indexes[i] = word.index + * indexes[j] = word.index */ - (__pyx_v_codelens[__pyx_v_i]) = 0; - goto __pyx_L7; + goto __pyx_L11_continue; } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":728 - * codelens[i] = 0 + /* "trunk/gensim/models/doc2vec_inner.pyx":481 + * continue # leaving j unchanged * else: - * indexes[i] = word.index # <<<<<<<<<<<<<< + * indexes[j] = word.index # <<<<<<<<<<<<<< * if hs: - * codelens[i] = len(word.code) + * codelens[j] = len(word.code) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 728; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 481; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 728; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 481; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - (__pyx_v_indexes[__pyx_v_i]) = __pyx_t_13; + (__pyx_v_indexes[__pyx_v_j]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":729 + /* "trunk/gensim/models/doc2vec_inner.pyx":482 * else: - * indexes[i] = word.index + * indexes[j] = word.index * if hs: # <<<<<<<<<<<<<< - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) */ - __pyx_t_12 = (__pyx_v_hs != 0); - if (__pyx_t_12) { + __pyx_t_5 = (__pyx_v_hs != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":730 - * indexes[i] = word.index + /* "trunk/gensim/models/doc2vec_inner.pyx":483 + * indexes[j] = word.index * if hs: - * codelens[i] = len(word.code) # <<<<<<<<<<<<<< - * codes[i] = np.PyArray_DATA(word.code) - * points[i] = np.PyArray_DATA(word.point) + * codelens[j] = len(word.code) # <<<<<<<<<<<<<< + * codes[j] = np.PyArray_DATA(word.code) + * points[j] = np.PyArray_DATA(word.point) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 730; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 483; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_5 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 730; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_12 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_12 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 483; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - (__pyx_v_codelens[__pyx_v_i]) = ((int)__pyx_t_5); + (__pyx_v_codelens[__pyx_v_j]) = ((int)__pyx_t_12); - /* "trunk/gensim/models/doc2vec_inner.pyx":731 + /* "trunk/gensim/models/doc2vec_inner.pyx":484 * if hs: - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< - * points[i] = np.PyArray_DATA(word.point) - * else: + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< + * points[j] = np.PyArray_DATA(word.point) + * result += 1 */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 731; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 484; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 731; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_codes[__pyx_v_i]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 484; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_codes[__pyx_v_j]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":732 - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) - * points[i] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< - * else: - * codelens[i] = 1 + /* "trunk/gensim/models/doc2vec_inner.pyx":485 + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) + * points[j] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< + * result += 1 + * j = j + 1 */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 732; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 485; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 732; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_points[__pyx_v_i]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 485; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_points[__pyx_v_j]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - goto __pyx_L8; + goto __pyx_L14; } - /*else*/ { + __pyx_L14:; - /* "trunk/gensim/models/doc2vec_inner.pyx":734 - * points[i] = np.PyArray_DATA(word.point) - * else: - * codelens[i] = 1 # <<<<<<<<<<<<<< - * result += 1 + /* "trunk/gensim/models/doc2vec_inner.pyx":486 + * codes[j] = np.PyArray_DATA(word.code) + * points[j] = np.PyArray_DATA(word.point) + * result += 1 # <<<<<<<<<<<<<< + * j = j + 1 * # single randint() call avoids a big thread-sync slowdown */ - (__pyx_v_codelens[__pyx_v_i]) = 1; - } - __pyx_L8:; + __pyx_v_result = (__pyx_v_result + 1); - /* "trunk/gensim/models/doc2vec_inner.pyx":735 - * else: - * codelens[i] = 1 - * result += 1 # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":487 + * points[j] = np.PyArray_DATA(word.point) + * result += 1 + * j = j + 1 # <<<<<<<<<<<<<< * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): + * for i, item in enumerate(np.random.randint(0, window, document_len)): */ - __pyx_v_result = (__pyx_v_result + 1); + __pyx_v_j = (__pyx_v_j + 1); } - __pyx_L7:; + __pyx_L11_continue:; } - /* "trunk/gensim/models/doc2vec_inner.pyx":737 - * result += 1 + /* "trunk/gensim/models/doc2vec_inner.pyx":489 + * j = j + 1 * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< + * for i, item in enumerate(np.random.randint(0, window, document_len)): # <<<<<<<<<<<<<< * reduced_windows[i] = item - * for i in range(lbl_length): + * */ __pyx_t_2 = 0; - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_14); + __pyx_t_10 = __Pyx_PyInt_From_int(__pyx_v_document_len); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); __pyx_t_15 = NULL; - __pyx_t_5 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_15 = PyMethod_GET_SELF(__pyx_t_7); + __pyx_t_12 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_15 = PyMethod_GET_SELF(__pyx_t_8); if (likely(__pyx_t_15)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); __Pyx_INCREF(__pyx_t_15); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - __pyx_t_5 = 1; + __Pyx_DECREF_SET(__pyx_t_8, function); + __pyx_t_12 = 1; } } - __pyx_t_16 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_16 = PyTuple_New(3+__pyx_t_12); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_15) { PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_15); __Pyx_GIVEREF(__pyx_t_15); __pyx_t_15 = NULL; } __Pyx_INCREF(__pyx_int_0); - PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_5, __pyx_int_0); + PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_12, __pyx_int_0); __Pyx_GIVEREF(__pyx_int_0); - PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_5, __pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_12, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_16, 2+__pyx_t_5, __pyx_t_14); - __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_16, 2+__pyx_t_12, __pyx_t_10); + __Pyx_GIVEREF(__pyx_t_10); __pyx_t_1 = 0; - __pyx_t_14 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_10 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (likely(PyList_CheckExact(__pyx_t_6)) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); __pyx_t_5 = 0; + __pyx_t_8 = __pyx_t_6; __Pyx_INCREF(__pyx_t_8); __pyx_t_12 = 0; __pyx_t_17 = NULL; } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_12 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_17 = Py_TYPE(__pyx_t_8)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; for (;;) { if (likely(!__pyx_t_17)) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_12 >= PyList_GET_SIZE(__pyx_t_8)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_12); __Pyx_INCREF(__pyx_t_6); __pyx_t_12++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_8, __pyx_t_12); __pyx_t_12++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } else { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + if (__pyx_t_12 >= PyTuple_GET_SIZE(__pyx_t_8)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_12); __Pyx_INCREF(__pyx_t_6); __pyx_t_12++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_8, __pyx_t_12); __pyx_t_12++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } } else { - __pyx_t_6 = __pyx_t_17(__pyx_t_7); + __pyx_t_6 = __pyx_t_17(__pyx_t_8); if (unlikely(!__pyx_t_6)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 489; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } break; } __Pyx_GOTREF(__pyx_t_6); } - __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_v_i = __pyx_t_2; - __pyx_t_2 = (__pyx_t_2 + 1); - - /* "trunk/gensim/models/doc2vec_inner.pyx":738 - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): - * reduced_windows[i] = item # <<<<<<<<<<<<<< - * for i in range(lbl_length): - * word = lbls[i] - */ - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 738; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_13; - - /* "trunk/gensim/models/doc2vec_inner.pyx":737 - * result += 1 - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< - * reduced_windows[i] = item - * for i in range(lbl_length): - */ - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - - /* "trunk/gensim/models/doc2vec_inner.pyx":739 - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): - * reduced_windows[i] = item - * for i in range(lbl_length): # <<<<<<<<<<<<<< - * word = lbls[i] - * if word is None: - */ - __pyx_t_2 = __pyx_v_lbl_length; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_i = __pyx_t_11; - - /* "trunk/gensim/models/doc2vec_inner.pyx":740 - * reduced_windows[i] = item - * for i in range(lbl_length): - * word = lbls[i] # <<<<<<<<<<<<<< - * if word is None: - * lbl_codelens[i] = 0 - */ - __pyx_t_7 = __Pyx_GetItemInt(__pyx_v_lbls, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_7 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 740; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - __Pyx_GOTREF(__pyx_t_7); - __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_7); - __pyx_t_7 = 0; - - /* "trunk/gensim/models/doc2vec_inner.pyx":741 - * for i in range(lbl_length): - * word = lbls[i] - * if word is None: # <<<<<<<<<<<<<< - * lbl_codelens[i] = 0 - * else: - */ - __pyx_t_12 = (__pyx_v_word == Py_None); - __pyx_t_4 = (__pyx_t_12 != 0); - if (__pyx_t_4) { + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_v_i = __pyx_t_2; + __pyx_t_2 = (__pyx_t_2 + 1); - /* "trunk/gensim/models/doc2vec_inner.pyx":742 - * word = lbls[i] - * if word is None: - * lbl_codelens[i] = 0 # <<<<<<<<<<<<<< - * else: - * lbl_indexes[i] = word.index + /* "trunk/gensim/models/doc2vec_inner.pyx":490 + * # single randint() call avoids a big thread-sync slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): + * reduced_windows[i] = item # <<<<<<<<<<<<<< + * + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) */ - (__pyx_v_lbl_codelens[__pyx_v_i]) = 0; - goto __pyx_L13; - } - /*else*/ { + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 490; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":744 - * lbl_codelens[i] = 0 - * else: - * lbl_indexes[i] = word.index # <<<<<<<<<<<<<< - * if hs: - * lbl_codelens[i] = len(word.code) + /* "trunk/gensim/models/doc2vec_inner.pyx":489 + * j = j + 1 + * # single randint() call avoids a big thread-sync slowdown + * for i, item in enumerate(np.random.randint(0, window, document_len)): # <<<<<<<<<<<<<< + * reduced_windows[i] = item + * */ - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 744; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_7); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 744; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - (__pyx_v_lbl_indexes[__pyx_v_i]) = __pyx_t_13; + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":745 - * else: - * lbl_indexes[i] = word.index - * if hs: # <<<<<<<<<<<<<< - * lbl_codelens[i] = len(word.code) - * else: + /* "trunk/gensim/models/doc2vec_inner.pyx":492 + * reduced_windows[i] = item + * + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) # <<<<<<<<<<<<<< + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] */ - __pyx_t_4 = (__pyx_v_hs != 0); - if (__pyx_t_4) { + __pyx_t_12 = PyObject_Length(__pyx_v_doctag_indexes); if (unlikely(__pyx_t_12 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 492; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_11 = 10000; + if (((__pyx_t_12 < __pyx_t_11) != 0)) { + __pyx_t_7 = __pyx_t_12; + } else { + __pyx_t_7 = __pyx_t_11; + } + __pyx_v_doctag_len = ((int)__pyx_t_7); - /* "trunk/gensim/models/doc2vec_inner.pyx":746 - * lbl_indexes[i] = word.index - * if hs: - * lbl_codelens[i] = len(word.code) # <<<<<<<<<<<<<< - * else: - * lbl_codelens[i] = 1 + /* "trunk/gensim/models/doc2vec_inner.pyx":493 + * + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + * for i in range(doctag_len): # <<<<<<<<<<<<<< + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 */ - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 746; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_5 = PyObject_Length(__pyx_t_7); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 746; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - (__pyx_v_lbl_codelens[__pyx_v_i]) = ((int)__pyx_t_5); - goto __pyx_L14; - } - /*else*/ { + __pyx_t_2 = __pyx_v_doctag_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":748 - * lbl_codelens[i] = len(word.code) - * else: - * lbl_codelens[i] = 1 # <<<<<<<<<<<<<< - * result += 1 + /* "trunk/gensim/models/doc2vec_inner.pyx":494 + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] # <<<<<<<<<<<<<< + * result += 1 * */ - (__pyx_v_lbl_codelens[__pyx_v_i]) = 1; - } - __pyx_L14:; + __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_doctag_indexes, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_8 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 494; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_8); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 494; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + (__pyx_v__doctag_indexes[__pyx_v_i]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":749 - * else: - * lbl_codelens[i] = 1 - * result += 1 # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":495 + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 # <<<<<<<<<<<<<< * - * # release GIL & train on the sentence + * # release GIL & train on the document */ - __pyx_v_result = (__pyx_v_result + 1); - } - __pyx_L13:; + __pyx_v_result = (__pyx_v_result + 1); } - /* "trunk/gensim/models/doc2vec_inner.pyx":752 + /* "trunk/gensim/models/doc2vec_inner.pyx":498 * - * # release GIL & train on the sentence + * # release GIL & train on the document * with nogil: # <<<<<<<<<<<<<< - * for j in range(lbl_length): - * if lbl_codelens[j] == 0: + * for i in range(document_len): + * j = i - window + reduced_windows[i] */ { #ifdef WITH_THREAD @@ -6996,208 +5308,524 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_train_sentence_ #endif /*try:*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":753 - * # release GIL & train on the sentence + /* "trunk/gensim/models/doc2vec_inner.pyx":499 + * # release GIL & train on the document * with nogil: - * for j in range(lbl_length): # <<<<<<<<<<<<<< - * if lbl_codelens[j] == 0: - * continue + * for i in range(document_len): # <<<<<<<<<<<<<< + * j = i - window + reduced_windows[i] + * if j < 0: */ - __pyx_t_2 = __pyx_v_lbl_length; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_j = __pyx_t_11; + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":754 + /* "trunk/gensim/models/doc2vec_inner.pyx":500 * with nogil: - * for j in range(lbl_length): - * if lbl_codelens[j] == 0: # <<<<<<<<<<<<<< - * continue - * for i in range(sentence_len): + * for i in range(document_len): + * j = i - window + reduced_windows[i] # <<<<<<<<<<<<<< + * if j < 0: + * j = 0 */ - __pyx_t_4 = (((__pyx_v_lbl_codelens[__pyx_v_j]) == 0) != 0); - if (__pyx_t_4) { + __pyx_v_j = ((__pyx_v_i - __pyx_v_window) + (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/doc2vec_inner.pyx":755 - * for j in range(lbl_length): - * if lbl_codelens[j] == 0: - * continue # <<<<<<<<<<<<<< - * for i in range(sentence_len): - * if codelens[i] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":501 + * for i in range(document_len): + * j = i - window + reduced_windows[i] + * if j < 0: # <<<<<<<<<<<<<< + * j = 0 + * k = i + window + 1 - reduced_windows[i] + */ + __pyx_t_5 = ((__pyx_v_j < 0) != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":502 + * j = i - window + reduced_windows[i] + * if j < 0: + * j = 0 # <<<<<<<<<<<<<< + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: */ - goto __pyx_L18_continue; + __pyx_v_j = 0; + goto __pyx_L24; } + __pyx_L24:; - /* "trunk/gensim/models/doc2vec_inner.pyx":756 - * if lbl_codelens[j] == 0: - * continue - * for i in range(sentence_len): # <<<<<<<<<<<<<< - * if codelens[i] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":503 + * if j < 0: + * j = 0 + * k = i + window + 1 - reduced_windows[i] # <<<<<<<<<<<<<< + * if k > document_len: + * k = document_len + */ + __pyx_v_k = (((__pyx_v_i + __pyx_v_window) + 1) - (__pyx_v_reduced_windows[__pyx_v_i])); + + /* "trunk/gensim/models/doc2vec_inner.pyx":504 + * j = 0 + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: # <<<<<<<<<<<<<< + * k = document_len + * + */ + __pyx_t_5 = ((__pyx_v_k > __pyx_v_document_len) != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":505 + * k = i + window + 1 - reduced_windows[i] + * if k > document_len: + * k = document_len # <<<<<<<<<<<<<< + * + * # compose l1 (in _neu1) & clear _work + */ + __pyx_v_k = __pyx_v_document_len; + goto __pyx_L25; + } + __pyx_L25:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":508 + * + * # compose l1 (in _neu1) & clear _work + * memset(_neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< + * count = 0.0 + * for m in range(j, k): + */ + memset(__pyx_v__neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":509 + * # compose l1 (in _neu1) & clear _work + * memset(_neu1, 0, size * cython.sizeof(REAL_t)) + * count = 0.0 # <<<<<<<<<<<<<< + * for m in range(j, k): + * if m == i: + */ + __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.0); + + /* "trunk/gensim/models/doc2vec_inner.pyx":510 + * memset(_neu1, 0, size * cython.sizeof(REAL_t)) + * count = 0.0 + * for m in range(j, k): # <<<<<<<<<<<<<< + * if m == i: * continue */ - __pyx_t_18 = __pyx_v_sentence_len; - for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { - __pyx_v_i = __pyx_t_19; + __pyx_t_18 = __pyx_v_k; + for (__pyx_t_19 = __pyx_v_j; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { + __pyx_v_m = __pyx_t_19; - /* "trunk/gensim/models/doc2vec_inner.pyx":757 - * continue - * for i in range(sentence_len): - * if codelens[i] == 0: # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":511 + * count = 0.0 + * for m in range(j, k): + * if m == i: # <<<<<<<<<<<<<< * continue - * if hs: + * else: */ - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_i]) == 0) != 0); - if (__pyx_t_4) { + __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":758 - * for i in range(sentence_len): - * if codelens[i] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":512 + * for m in range(j, k): + * if m == i: * continue # <<<<<<<<<<<<<< - * if hs: - * fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) + * else: + * count += ONEF */ - goto __pyx_L21_continue; + goto __pyx_L26_continue; } + /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":759 - * if codelens[i] == 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":514 * continue - * if hs: # <<<<<<<<<<<<<< - * fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) - * if negative: + * else: + * count += ONEF # <<<<<<<<<<<<<< + * our_saxpy(&size, &ONEF, &_word_vectors[indexes[m] * size], &ONE, _neu1, &ONE) + * for m in range(doctag_len): */ - __pyx_t_4 = (__pyx_v_hs != 0); - if (__pyx_t_4) { + __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); - /* "trunk/gensim/models/doc2vec_inner.pyx":760 - * continue - * if hs: - * fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) # <<<<<<<<<<<<<< - * if negative: - * next_random = fast_sentence_dbow_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], lbl_indexes[j], _alpha, work, next_random, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":515 + * else: + * count += ONEF + * our_saxpy(&size, &ONEF, &_word_vectors[indexes[m] * size], &ONE, _neu1, &ONE) # <<<<<<<<<<<<<< + * for m in range(doctag_len): + * count += ONEF */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, (__pyx_v_lbl_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work, __pyx_v_tw, __pyx_v_tl); - goto __pyx_L24; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v__word_vectors[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v__neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); } - __pyx_L24:; + __pyx_L26_continue:; + } - /* "trunk/gensim/models/doc2vec_inner.pyx":761 - * if hs: - * fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) - * if negative: # <<<<<<<<<<<<<< - * next_random = fast_sentence_dbow_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], lbl_indexes[j], _alpha, work, next_random, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":516 + * count += ONEF + * our_saxpy(&size, &ONEF, &_word_vectors[indexes[m] * size], &ONE, _neu1, &ONE) + * for m in range(doctag_len): # <<<<<<<<<<<<<< + * count += ONEF + * our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) + */ + __pyx_t_18 = __pyx_v_doctag_len; + for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { + __pyx_v_m = __pyx_t_19; + + /* "trunk/gensim/models/doc2vec_inner.pyx":517 + * our_saxpy(&size, &ONEF, &_word_vectors[indexes[m] * size], &ONE, _neu1, &ONE) + * for m in range(doctag_len): + * count += ONEF # <<<<<<<<<<<<<< + * our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) + * if count > (0.5): + */ + __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF); + + /* "trunk/gensim/models/doc2vec_inner.pyx":518 + * for m in range(doctag_len): + * count += ONEF + * our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) # <<<<<<<<<<<<<< + * if count > (0.5): + * inv_count = ONEF/count + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF), (&(__pyx_v__doctag_vectors[((__pyx_v__doctag_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v__neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":519 + * count += ONEF + * our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) + * if count > (0.5): # <<<<<<<<<<<<<< + * inv_count = ONEF/count + * if cbow_mean: + */ + __pyx_t_5 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)0.5)) != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":520 + * our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) + * if count > (0.5): + * inv_count = ONEF/count # <<<<<<<<<<<<<< + * if cbow_mean: + * sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) + */ + __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF / __pyx_v_count); + goto __pyx_L31; + } + __pyx_L31:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":521 + * if count > (0.5): + * inv_count = ONEF/count + * if cbow_mean: # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) + * memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error + */ + __pyx_t_5 = (__pyx_v_cbow_mean != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":522 + * inv_count = ONEF/count + * if cbow_mean: + * sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< + * memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error + * if hs: + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v__neu1, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L32; + } + __pyx_L32:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":523 + * if cbow_mean: + * sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) + * memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error # <<<<<<<<<<<<<< + * if hs: + * fast_document_dm_hs(points[i], codes[i], codelens[i], + */ + memset(__pyx_v__work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":524 + * sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) + * memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error + * if hs: # <<<<<<<<<<<<<< + * fast_document_dm_hs(points[i], codes[i], codelens[i], + * _neu1, syn1, _alpha, _work, + */ + __pyx_t_5 = (__pyx_v_hs != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":525 + * memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error + * if hs: + * fast_document_dm_hs(points[i], codes[i], codelens[i], # <<<<<<<<<<<<<< + * _neu1, syn1, _alpha, _work, + * size, _learn_hidden) + */ + __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v__neu1, __pyx_v_syn1, __pyx_v__alpha, __pyx_v__work, __pyx_v_size, __pyx_v__learn_hidden); + goto __pyx_L33; + } + __pyx_L33:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":528 + * _neu1, syn1, _alpha, _work, + * size, _learn_hidden) + * if negative: # <<<<<<<<<<<<<< + * next_random = fast_document_dm_neg(negative, table, table_len, next_random, + * _neu1, syn1neg, indexes[i], _alpha, _work, + */ + __pyx_t_5 = (__pyx_v_negative != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":529 + * size, _learn_hidden) + * if negative: + * next_random = fast_document_dm_neg(negative, table, table_len, next_random, # <<<<<<<<<<<<<< + * _neu1, syn1neg, indexes[i], _alpha, _work, + * size, _learn_hidden) + */ + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dm_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_next_random, __pyx_v__neu1, __pyx_v_syn1neg, (__pyx_v_indexes[__pyx_v_i]), __pyx_v__alpha, __pyx_v__work, __pyx_v_size, __pyx_v__learn_hidden); + goto __pyx_L34; + } + __pyx_L34:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":533 + * size, _learn_hidden) * + * if not cbow_mean: # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, _work, &ONE) # (does this need BLAS-variants like saxpy?) + * # apply accumulated error in work */ - __pyx_t_4 = (__pyx_v_negative != 0); - if (__pyx_t_4) { + __pyx_t_5 = ((!(__pyx_v_cbow_mean != 0)) != 0); + if (__pyx_t_5) { - /* "trunk/gensim/models/doc2vec_inner.pyx":762 - * fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) - * if negative: - * next_random = fast_sentence_dbow_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], lbl_indexes[j], _alpha, work, next_random, tw, tl) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":534 * - * return result + * if not cbow_mean: + * sscal(&size, &inv_count, _work, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< + * # apply accumulated error in work + * if _learn_doctags: */ - __pyx_v_next_random = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, (__pyx_v_indexes[__pyx_v_i]), (__pyx_v_lbl_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work, __pyx_v_next_random, __pyx_v_tw, __pyx_v_tl); - goto __pyx_L25; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v__work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + goto __pyx_L35; + } + __pyx_L35:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":536 + * sscal(&size, &inv_count, _work, &ONE) # (does this need BLAS-variants like saxpy?) + * # apply accumulated error in work + * if _learn_doctags: # <<<<<<<<<<<<<< + * for m in range(doctag_len): + * our_saxpy(&size, &_doctag_locks[_doctag_indexes[m]], _work, + */ + __pyx_t_5 = (__pyx_v__learn_doctags != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":537 + * # apply accumulated error in work + * if _learn_doctags: + * for m in range(doctag_len): # <<<<<<<<<<<<<< + * our_saxpy(&size, &_doctag_locks[_doctag_indexes[m]], _work, + * &ONE, &_doctag_vectors[_doctag_indexes[m] * size], &ONE) + */ + __pyx_t_18 = __pyx_v_doctag_len; + for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { + __pyx_v_m = __pyx_t_19; + + /* "trunk/gensim/models/doc2vec_inner.pyx":538 + * if _learn_doctags: + * for m in range(doctag_len): + * our_saxpy(&size, &_doctag_locks[_doctag_indexes[m]], _work, # <<<<<<<<<<<<<< + * &ONE, &_doctag_vectors[_doctag_indexes[m] * size], &ONE) + * if _learn_words: + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v__doctag_locks[(__pyx_v__doctag_indexes[__pyx_v_m])])), __pyx_v__work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v__doctag_vectors[((__pyx_v__doctag_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + } + goto __pyx_L36; + } + __pyx_L36:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":540 + * our_saxpy(&size, &_doctag_locks[_doctag_indexes[m]], _work, + * &ONE, &_doctag_vectors[_doctag_indexes[m] * size], &ONE) + * if _learn_words: # <<<<<<<<<<<<<< + * for m in range(j, k): + * if m == i: + */ + __pyx_t_5 = (__pyx_v__learn_words != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":541 + * &ONE, &_doctag_vectors[_doctag_indexes[m] * size], &ONE) + * if _learn_words: + * for m in range(j, k): # <<<<<<<<<<<<<< + * if m == i: + * continue + */ + __pyx_t_18 = __pyx_v_k; + for (__pyx_t_19 = __pyx_v_j; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { + __pyx_v_m = __pyx_t_19; + + /* "trunk/gensim/models/doc2vec_inner.pyx":542 + * if _learn_words: + * for m in range(j, k): + * if m == i: # <<<<<<<<<<<<<< + * continue + * else: + */ + __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); + if (__pyx_t_5) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":543 + * for m in range(j, k): + * if m == i: + * continue # <<<<<<<<<<<<<< + * else: + * our_saxpy(&size, &_word_locks[indexes[m]], _work, &ONE, + */ + goto __pyx_L40_continue; + } + /*else*/ { + + /* "trunk/gensim/models/doc2vec_inner.pyx":545 + * continue + * else: + * our_saxpy(&size, &_word_locks[indexes[m]], _work, &ONE, # <<<<<<<<<<<<<< + * &_word_vectors[indexes[m] * size], &ONE) + * + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v__word_locks[(__pyx_v_indexes[__pyx_v_m])])), __pyx_v__work, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v__word_vectors[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + } + __pyx_L40_continue:; } - __pyx_L25:; - __pyx_L21_continue:; + goto __pyx_L39; } - __pyx_L18_continue:; + __pyx_L39:; } } - /* "trunk/gensim/models/doc2vec_inner.pyx":752 + /* "trunk/gensim/models/doc2vec_inner.pyx":498 * - * # release GIL & train on the sentence + * # release GIL & train on the document * with nogil: # <<<<<<<<<<<<<< - * for j in range(lbl_length): - * if lbl_codelens[j] == 0: + * for i in range(document_len): + * j = i - window + reduced_windows[i] */ /*finally:*/ { /*normal exit:*/{ #ifdef WITH_THREAD Py_BLOCK_THREADS #endif - goto __pyx_L17; + goto __pyx_L21; } - __pyx_L17:; + __pyx_L21:; } } - /* "trunk/gensim/models/doc2vec_inner.pyx":764 - * next_random = fast_sentence_dbow_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], lbl_indexes[j], _alpha, work, next_random, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":548 + * &_word_vectors[indexes[m] * size], &ONE) * * return result # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 764; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_t_8 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 548; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":675 - * return next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":399 * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_14); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __Pyx_XDECREF(__pyx_t_15); __Pyx_XDECREF(__pyx_t_16); - __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_sentence_dbow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dm", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF(__pyx_v_word); __Pyx_XDECREF(__pyx_v_item); + __Pyx_XDECREF(__pyx_v_work); + __Pyx_XDECREF(__pyx_v_neu1); + __Pyx_XDECREF(__pyx_v_word_vectors); + __Pyx_XDECREF(__pyx_v_word_locks); + __Pyx_XDECREF(__pyx_v_doctag_vectors); + __Pyx_XDECREF(__pyx_v_doctag_locks); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":767 +/* "trunk/gensim/models/doc2vec_inner.pyx":551 * * - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ /* Python wrapper */ -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence_dm(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence_dm = {"train_sentence_dm", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence_dm, METH_VARARGS|METH_KEYWORDS, 0}; -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence_dm(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5train_document_dm_concat(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_5train_document_dm_concat = {"train_document_dm_concat", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5train_document_dm_concat, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5train_document_dm_concat(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_model = 0; - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_lbls = 0; + PyObject *__pyx_v_word_vocabs = 0; + PyObject *__pyx_v_doctag_indexes = 0; PyObject *__pyx_v_alpha = 0; - PyObject *__pyx_v__work = 0; - PyObject *__pyx_v__neu1 = 0; - PyObject *__pyx_v_train_words = 0; - PyObject *__pyx_v_train_lbls = 0; + PyObject *__pyx_v_work = 0; + PyObject *__pyx_v_neu1 = 0; + PyObject *__pyx_v_learn_doctags = 0; + PyObject *__pyx_v_learn_words = 0; + PyObject *__pyx_v_learn_hidden = 0; + PyObject *__pyx_v_word_vectors = 0; + PyObject *__pyx_v_word_locks = 0; + PyObject *__pyx_v_doctag_vectors = 0; + PyObject *__pyx_v_doctag_locks = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("train_sentence_dm (wrapper)", 0); + __Pyx_RefNannySetupContext("train_document_dm_concat (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_model,&__pyx_n_s_sentence,&__pyx_n_s_lbls,&__pyx_n_s_alpha,&__pyx_n_s_work,&__pyx_n_s_neu1,&__pyx_n_s_train_words,&__pyx_n_s_train_lbls,0}; - PyObject* values[8] = {0,0,0,0,0,0,0,0}; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_model,&__pyx_n_s_word_vocabs,&__pyx_n_s_doctag_indexes,&__pyx_n_s_alpha,&__pyx_n_s_work,&__pyx_n_s_neu1,&__pyx_n_s_learn_doctags,&__pyx_n_s_learn_words,&__pyx_n_s_learn_hidden,&__pyx_n_s_word_vectors,&__pyx_n_s_word_locks,&__pyx_n_s_doctag_vectors,&__pyx_n_s_doctag_locks,0}; + PyObject* values[13] = {0,0,0,0,0,0,0,0,0,0,0,0,0}; + values[4] = ((PyObject *)Py_None); + values[5] = ((PyObject *)Py_None); + + /* "trunk/gensim/models/doc2vec_inner.pyx":552 + * + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + * learn_doctags=True, learn_words=True, learn_hidden=True, # <<<<<<<<<<<<<< + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + * cdef int hs = model.hs + */ + values[6] = ((PyObject *)Py_True); + values[7] = ((PyObject *)Py_True); + values[8] = ((PyObject *)Py_True); + + /* "trunk/gensim/models/doc2vec_inner.pyx":553 + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): # <<<<<<<<<<<<<< + * cdef int hs = model.hs + * cdef int negative = model.negative + */ + values[9] = ((PyObject *)Py_None); + values[10] = ((PyObject *)Py_None); + values[11] = ((PyObject *)Py_None); + values[12] = ((PyObject *)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); @@ -7215,103 +5843,155 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_model)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vocabs)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm_concat", 0, 4, 13, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_lbls)) != 0)) kw_args--; + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_indexes)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm_concat", 0, 4, 13, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_alpha)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm_concat", 0, 4, 13, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 4: - if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work); + if (value) { values[4] = value; kw_args--; } } case 5: - if (likely((values[5] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_neu1)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 5); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_neu1); + if (value) { values[5] = value; kw_args--; } } case 6: - if (likely((values[6] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_train_words)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 6); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_doctags); + if (value) { values[6] = value; kw_args--; } } case 7: - if (likely((values[7] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_train_lbls)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, 7); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_words); + if (value) { values[7] = value; kw_args--; } + } + case 8: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_learn_hidden); + if (value) { values[8] = value; kw_args--; } + } + case 9: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_vectors); + if (value) { values[9] = value; kw_args--; } + } + case 10: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word_locks); + if (value) { values[10] = value; kw_args--; } + } + case 11: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_vectors); + if (value) { values[11] = value; kw_args--; } + } + case 12: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_doctag_locks); + if (value) { values[12] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_dm") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_document_dm_concat") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } - } else if (PyTuple_GET_SIZE(__pyx_args) != 8) { - goto __pyx_L5_argtuple_error; } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - values[3] = PyTuple_GET_ITEM(__pyx_args, 3); - values[4] = PyTuple_GET_ITEM(__pyx_args, 4); - values[5] = PyTuple_GET_ITEM(__pyx_args, 5); - values[6] = PyTuple_GET_ITEM(__pyx_args, 6); - values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 13: values[12] = PyTuple_GET_ITEM(__pyx_args, 12); + case 12: values[11] = PyTuple_GET_ITEM(__pyx_args, 11); + case 11: values[10] = PyTuple_GET_ITEM(__pyx_args, 10); + case 10: values[9] = PyTuple_GET_ITEM(__pyx_args, 9); + case 9: values[8] = PyTuple_GET_ITEM(__pyx_args, 8); + case 8: values[7] = PyTuple_GET_ITEM(__pyx_args, 7); + case 7: values[6] = PyTuple_GET_ITEM(__pyx_args, 6); + case 6: values[5] = PyTuple_GET_ITEM(__pyx_args, 5); + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } } __pyx_v_model = values[0]; - __pyx_v_sentence = values[1]; - __pyx_v_lbls = values[2]; + __pyx_v_word_vocabs = values[1]; + __pyx_v_doctag_indexes = values[2]; __pyx_v_alpha = values[3]; - __pyx_v__work = values[4]; - __pyx_v__neu1 = values[5]; - __pyx_v_train_words = values[6]; - __pyx_v_train_lbls = values[7]; + __pyx_v_work = values[4]; + __pyx_v_neu1 = values[5]; + __pyx_v_learn_doctags = values[6]; + __pyx_v_learn_words = values[7]; + __pyx_v_learn_hidden = values[8]; + __pyx_v_word_vectors = values[9]; + __pyx_v_word_locks = values[10]; + __pyx_v_doctag_vectors = values[11]; + __pyx_v_doctag_locks = values[12]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("train_sentence_dm", 1, 8, 8, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_document_dm_concat", 0, 4, 13, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; - __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_sentence_dm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dm_concat", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence_dm(__pyx_self, __pyx_v_model, __pyx_v_sentence, __pyx_v_lbls, __pyx_v_alpha, __pyx_v__work, __pyx_v__neu1, __pyx_v_train_words, __pyx_v_train_lbls); + __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4train_document_dm_concat(__pyx_self, __pyx_v_model, __pyx_v_word_vocabs, __pyx_v_doctag_indexes, __pyx_v_alpha, __pyx_v_work, __pyx_v_neu1, __pyx_v_learn_doctags, __pyx_v_learn_words, __pyx_v_learn_hidden, __pyx_v_word_vectors, __pyx_v_word_locks, __pyx_v_doctag_vectors, __pyx_v_doctag_locks); + + /* "trunk/gensim/models/doc2vec_inner.pyx":551 + * + * + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + */ /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence_dm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_sentence, PyObject *__pyx_v_lbls, PyObject *__pyx_v_alpha, PyObject *__pyx_v__work, PyObject *__pyx_v__neu1, PyObject *__pyx_v_train_words, PyObject *__pyx_v_train_lbls) { +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4train_document_dm_concat(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_word_vocabs, PyObject *__pyx_v_doctag_indexes, PyObject *__pyx_v_alpha, PyObject *__pyx_v_work, PyObject *__pyx_v_neu1, PyObject *__pyx_v_learn_doctags, PyObject *__pyx_v_learn_words, PyObject *__pyx_v_learn_hidden, PyObject *__pyx_v_word_vectors, PyObject *__pyx_v_word_locks, PyObject *__pyx_v_doctag_vectors, PyObject *__pyx_v_doctag_locks) { int __pyx_v_hs; int __pyx_v_negative; - int __pyx_v_tw; - int __pyx_v_tl; - int __pyx_v_cbow_mean; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn0; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_work; - __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_neu1; + int __pyx_v__learn_doctags; + int __pyx_v__learn_words; + int __pyx_v__learn_hidden; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_vectors; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__word_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__doctag_locks; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__work; + __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v__neu1; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_v__alpha; - int __pyx_v_size; + int __pyx_v_layer1_size; + int __pyx_v_vector_size; int __pyx_v_codelens[10000]; - int __pyx_v_lbl_codelens[10000]; __pyx_t_5numpy_uint32_t __pyx_v_indexes[10000]; - __pyx_t_5numpy_uint32_t __pyx_v_lbl_indexes[10000]; - __pyx_t_5numpy_uint32_t __pyx_v_reduced_windows[10000]; - int __pyx_v_sentence_len; - int __pyx_v_lbl_length; + __pyx_t_5numpy_uint32_t __pyx_v__doctag_indexes[10000]; + __pyx_t_5numpy_uint32_t __pyx_v_window_indexes[10000]; + int __pyx_v_document_len; + int __pyx_v_doctag_len; int __pyx_v_window; + int __pyx_v_expected_doctag_len; int __pyx_v_i; int __pyx_v_j; int __pyx_v_k; + int __pyx_v_m; + int __pyx_v_n; long __pyx_v_result; + int __pyx_v_null_word_index; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *__pyx_v_syn1; __pyx_t_5numpy_uint32_t *__pyx_v_points[10000]; __pyx_t_5numpy_uint8_t *__pyx_v_codes[10000]; @@ -7320,679 +6000,812 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence unsigned PY_LONG_LONG __pyx_v_table_len; unsigned PY_LONG_LONG __pyx_v_next_random; PyObject *__pyx_v_word = NULL; - PyObject *__pyx_v_item = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; int __pyx_t_2; __pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t __pyx_t_3; - int __pyx_t_4; + PyObject *__pyx_t_4 = NULL; Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - unsigned PY_LONG_LONG __pyx_t_8; - long __pyx_t_9; - Py_ssize_t __pyx_t_10; - int __pyx_t_11; - int __pyx_t_12; - __pyx_t_5numpy_uint32_t __pyx_t_13; - PyObject *__pyx_t_14 = NULL; - PyObject *__pyx_t_15 = NULL; - PyObject *__pyx_t_16 = NULL; - PyObject *(*__pyx_t_17)(PyObject *); + long __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + unsigned PY_LONG_LONG __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + __pyx_t_5numpy_uint32_t __pyx_t_14; + int __pyx_t_15; + int __pyx_t_16; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("train_sentence_dm", 0); - - /* "trunk/gensim/models/doc2vec_inner.pyx":768 - * - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): + __Pyx_RefNannySetupContext("train_document_dm_concat", 0); + __Pyx_INCREF(__pyx_v_work); + __Pyx_INCREF(__pyx_v_neu1); + __Pyx_INCREF(__pyx_v_word_vectors); + __Pyx_INCREF(__pyx_v_word_locks); + __Pyx_INCREF(__pyx_v_doctag_vectors); + __Pyx_INCREF(__pyx_v_doctag_locks); + + /* "trunk/gensim/models/doc2vec_inner.pyx":554 + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): * cdef int hs = model.hs # <<<<<<<<<<<<<< * cdef int negative = model.negative - * cdef int tw = train_words + * cdef int _learn_doctags = learn_doctags */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 768; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 554; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 768; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 554; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_hs = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":769 - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): + /* "trunk/gensim/models/doc2vec_inner.pyx":555 + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): * cdef int hs = model.hs * cdef int negative = model.negative # <<<<<<<<<<<<<< - * cdef int tw = train_words - * cdef int tl = train_lbls + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 769; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 555; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 769; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 555; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_negative = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":770 + /* "trunk/gensim/models/doc2vec_inner.pyx":556 * cdef int hs = model.hs * cdef int negative = model.negative - * cdef int tw = train_words # <<<<<<<<<<<<<< - * cdef int tl = train_lbls - * cdef int cbow_mean = model.cbow_mean + * cdef int _learn_doctags = learn_doctags # <<<<<<<<<<<<<< + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_train_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 770; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_tw = __pyx_t_2; + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_doctags); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 556; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_doctags = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":771 + /* "trunk/gensim/models/doc2vec_inner.pyx":557 * cdef int negative = model.negative - * cdef int tw = train_words - * cdef int tl = train_lbls # <<<<<<<<<<<<<< - * cdef int cbow_mean = model.cbow_mean + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words # <<<<<<<<<<<<<< + * cdef int _learn_hidden = learn_hidden * */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_train_lbls); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 771; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_tl = __pyx_t_2; + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_words); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 557; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_words = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":772 - * cdef int tw = train_words - * cdef int tl = train_lbls - * cdef int cbow_mean = model.cbow_mean # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":558 + * cdef int _learn_doctags = learn_doctags + * cdef int _learn_words = learn_words + * cdef int _learn_hidden = learn_hidden # <<<<<<<<<<<<<< + * + * cdef REAL_t *_word_vectors + */ + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_learn_hidden); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 558; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__learn_hidden = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":566 + * cdef REAL_t *_work + * cdef REAL_t *_neu1 + * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< + * cdef int layer1_size = model.layer1_size + * cdef int vector_size = model.vector_size + */ + __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 566; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__alpha = __pyx_t_3; + + /* "trunk/gensim/models/doc2vec_inner.pyx":567 + * cdef REAL_t *_neu1 + * cdef REAL_t _alpha = alpha + * cdef int layer1_size = model.layer1_size # <<<<<<<<<<<<<< + * cdef int vector_size = model.vector_size * - * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_cbow_mean); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 772; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 567; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 772; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 567; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_cbow_mean = __pyx_t_2; + __pyx_v_layer1_size = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":774 - * cdef int cbow_mean = model.cbow_mean + /* "trunk/gensim/models/doc2vec_inner.pyx":568 + * cdef REAL_t _alpha = alpha + * cdef int layer1_size = model.layer1_size + * cdef int vector_size = model.vector_size # <<<<<<<<<<<<<< * - * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) # <<<<<<<<<<<<<< - * cdef REAL_t *work - * cdef REAL_t *neu1 + * cdef int codelens[MAX_DOCUMENT_LEN] */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 774; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_vector_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 568; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 774; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_syn0 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 568; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_vector_size = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":777 - * cdef REAL_t *work - * cdef REAL_t *neu1 - * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< - * cdef int size = model.layer1_size + /* "trunk/gensim/models/doc2vec_inner.pyx":576 + * cdef int document_len + * cdef int doctag_len + * cdef int window = model.window # <<<<<<<<<<<<<< + * cdef int expected_doctag_len = model.dm_tag_count * */ - __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 777; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v__alpha = __pyx_t_3; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 576; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 576; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_window = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":778 - * cdef REAL_t *neu1 - * cdef REAL_t _alpha = alpha - * cdef int size = model.layer1_size # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":577 + * cdef int doctag_len + * cdef int window = model.window + * cdef int expected_doctag_len = model.dm_tag_count # <<<<<<<<<<<<<< + * + * cdef int i, j, k, m, n + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_dm_tag_count); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 577; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 577; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_expected_doctag_len = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":580 + * + * cdef int i, j, k, m, n + * cdef long result = 0 # <<<<<<<<<<<<<< + * cdef int null_word_index = model.vocab['\0'].index + * + */ + __pyx_v_result = 0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":581 + * cdef int i, j, k, m, n + * cdef long result = 0 + * cdef int null_word_index = model.vocab['\0'].index # <<<<<<<<<<<<<< + * + * # For hierarchical softmax + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_vocab); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = PyObject_GetItem(__pyx_t_1, __pyx_kp_s__5); if (unlikely(__pyx_t_4 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_index); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_null_word_index = __pyx_t_2; + + /* "trunk/gensim/models/doc2vec_inner.pyx":594 + * cdef unsigned long long next_random + * + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) # <<<<<<<<<<<<<< + * if doctag_len != expected_doctag_len: + * return 0 # skip doc without expected nmber of tags + */ + __pyx_t_5 = PyObject_Length(__pyx_v_doctag_indexes); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 594; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = 10000; + if (((__pyx_t_5 < __pyx_t_6) != 0)) { + __pyx_t_7 = __pyx_t_5; + } else { + __pyx_t_7 = __pyx_t_6; + } + __pyx_v_doctag_len = ((int)__pyx_t_7); + + /* "trunk/gensim/models/doc2vec_inner.pyx":595 + * + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + * if doctag_len != expected_doctag_len: # <<<<<<<<<<<<<< + * return 0 # skip doc without expected nmber of tags + * + */ + __pyx_t_8 = ((__pyx_v_doctag_len != __pyx_v_expected_doctag_len) != 0); + if (__pyx_t_8) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":596 + * doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + * if doctag_len != expected_doctag_len: + * return 0 # skip doc without expected nmber of tags # <<<<<<<<<<<<<< + * + * # default vectors, locks from syn0/doctag_syn0 + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_int_0); + __pyx_r = __pyx_int_0; + goto __pyx_L0; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":599 * - * cdef int codelens[MAX_SENTENCE_LEN] + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: # <<<<<<<<<<<<<< + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 778; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 778; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_size = __pyx_t_2; + __pyx_t_8 = (__pyx_v_word_vectors == Py_None); + __pyx_t_9 = (__pyx_t_8 != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":787 - * cdef int sentence_len - * cdef int lbl_length - * cdef int window = model.window # <<<<<<<<<<<<<< - * - * cdef int i, j, k + /* "trunk/gensim/models/doc2vec_inner.pyx":600 + * # default vectors, locks from syn0/doctag_syn0 + * if word_vectors is None: + * word_vectors = model.syn0 # <<<<<<<<<<<<<< + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 787; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 787; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_window = __pyx_t_2; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 600; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_word_vectors, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L4; + } + __pyx_L4:; - /* "trunk/gensim/models/doc2vec_inner.pyx":790 - * - * cdef int i, j, k - * cdef long result = 0 # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":601 + * if word_vectors is None: + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) # <<<<<<<<<<<<<< + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 + */ + if (!(likely(((__pyx_v_word_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 601; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_vectors))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":602 + * word_vectors = model.syn0 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: # <<<<<<<<<<<<<< + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + */ + __pyx_t_9 = (__pyx_v_doctag_vectors == Py_None); + __pyx_t_8 = (__pyx_t_9 != 0); + if (__pyx_t_8) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":603 + * _word_vectors = (np.PyArray_DATA(word_vectors)) + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 # <<<<<<<<<<<<<< + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 603; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_doctag_syn0); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 603; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_vectors, __pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":604 + * if doctag_vectors is None: + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) # <<<<<<<<<<<<<< + * if word_locks is None: + * word_locks = model.syn0_lockf + */ + if (!(likely(((__pyx_v_doctag_vectors) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_vectors, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 604; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_vectors = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_vectors))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":605 + * doctag_vectors = model.docvecs.doctag_syn0 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: # <<<<<<<<<<<<<< + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + */ + __pyx_t_8 = (__pyx_v_word_locks == Py_None); + __pyx_t_9 = (__pyx_t_8 != 0); + if (__pyx_t_9) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":606 + * _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + * if word_locks is None: + * word_locks = model.syn0_lockf # <<<<<<<<<<<<<< + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: + */ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0_lockf); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 606; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF_SET(__pyx_v_word_locks, __pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":607 + * if word_locks is None: + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) # <<<<<<<<<<<<<< + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + */ + if (!(likely(((__pyx_v_word_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_word_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 607; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__word_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_word_locks))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":608 + * word_locks = model.syn0_lockf + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: # <<<<<<<<<<<<<< + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) + */ + __pyx_t_9 = (__pyx_v_doctag_locks == Py_None); + __pyx_t_8 = (__pyx_t_9 != 0); + if (__pyx_t_8) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":609 + * _word_locks = (np.PyArray_DATA(word_locks)) + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf # <<<<<<<<<<<<<< + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) + * + */ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_docvecs); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 609; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_doctag_syn0_lockf); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 609; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF_SET(__pyx_v_doctag_locks, __pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L7; + } + __pyx_L7:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":610 + * if doctag_locks is None: + * doctag_locks = model.docvecs.doctag_syn0_lockf + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) # <<<<<<<<<<<<<< * - * # For hierarchical softmax + * if hs: */ - __pyx_v_result = 0; + if (!(likely(((__pyx_v_doctag_locks) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_doctag_locks, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 610; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__doctag_locks = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_doctag_locks))); - /* "trunk/gensim/models/doc2vec_inner.pyx":805 - * cdef unsigned long long next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":612 + * _doctag_locks = (np.PyArray_DATA(doctag_locks)) * * if hs: # <<<<<<<<<<<<<< * syn1 = (np.PyArray_DATA(model.syn1)) * */ - __pyx_t_4 = (__pyx_v_hs != 0); - if (__pyx_t_4) { + __pyx_t_8 = (__pyx_v_hs != 0); + if (__pyx_t_8) { - /* "trunk/gensim/models/doc2vec_inner.pyx":806 + /* "trunk/gensim/models/doc2vec_inner.pyx":613 * * if hs: * syn1 = (np.PyArray_DATA(model.syn1)) # <<<<<<<<<<<<<< * * if negative: */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 613; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 613; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - goto __pyx_L3; + goto __pyx_L8; } - __pyx_L3:; + __pyx_L8:; - /* "trunk/gensim/models/doc2vec_inner.pyx":808 + /* "trunk/gensim/models/doc2vec_inner.pyx":615 * syn1 = (np.PyArray_DATA(model.syn1)) * * if negative: # <<<<<<<<<<<<<< * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) */ - __pyx_t_4 = (__pyx_v_negative != 0); - if (__pyx_t_4) { + __pyx_t_8 = (__pyx_v_negative != 0); + if (__pyx_t_8) { - /* "trunk/gensim/models/doc2vec_inner.pyx":809 + /* "trunk/gensim/models/doc2vec_inner.pyx":616 * * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) # <<<<<<<<<<<<<< * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 809; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 809; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1neg = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":810 + /* "trunk/gensim/models/doc2vec_inner.pyx":617 * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) # <<<<<<<<<<<<<< * table_len = len(model.table) * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 810; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 617; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 810; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 617; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_table = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":811 + /* "trunk/gensim/models/doc2vec_inner.pyx":618 * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) # <<<<<<<<<<<<<< * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 811; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 618; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 811; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 618; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_table_len = __pyx_t_5; + __pyx_v_table_len = __pyx_t_7; - /* "trunk/gensim/models/doc2vec_inner.pyx":812 + /* "trunk/gensim/models/doc2vec_inner.pyx":619 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_random); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_10, __pyx_n_s_randint); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_10 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = PyNumber_Add(__pyx_t_1, __pyx_t_10); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_v_next_random = __pyx_t_8; - goto __pyx_L4; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_11 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_4); if (unlikely((__pyx_t_11 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_next_random = __pyx_t_11; + goto __pyx_L9; } - __pyx_L4:; + __pyx_L9:; - /* "trunk/gensim/models/doc2vec_inner.pyx":815 + /* "trunk/gensim/models/doc2vec_inner.pyx":622 * * # convert Python structures to primitive types, so we can release the GIL - * work = np.PyArray_DATA(_work) # <<<<<<<<<<<<<< - * neu1 = np.PyArray_DATA(_neu1) - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) + * if work is None: # <<<<<<<<<<<<<< + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) */ - if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 815; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_work = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__work))); + __pyx_t_8 = (__pyx_v_work == Py_None); + __pyx_t_9 = (__pyx_t_8 != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":816 + /* "trunk/gensim/models/doc2vec_inner.pyx":623 * # convert Python structures to primitive types, so we can release the GIL - * work = np.PyArray_DATA(_work) - * neu1 = np.PyArray_DATA(_neu1) # <<<<<<<<<<<<<< - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) # <<<<<<<<<<<<<< + * _work = np.PyArray_DATA(work) + * if neu1 is None: + */ + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_zeros); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_10); + __Pyx_GIVEREF(__pyx_t_10); + __pyx_t_10 = 0; + __pyx_t_10 = PyDict_New(); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_12 = __Pyx_GetModuleGlobalName(__pyx_n_s_REAL); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_12); + if (PyDict_SetItem(__pyx_t_10, __pyx_n_s_dtype, __pyx_t_12) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_1, __pyx_t_10); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF_SET(__pyx_v_work, __pyx_t_12); + __pyx_t_12 = 0; + goto __pyx_L10; + } + __pyx_L10:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":624 + * if work is None: + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) # <<<<<<<<<<<<<< + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) + */ + if (!(likely(((__pyx_v_work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 624; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__work = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_work))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":625 + * work = zeros(model.layer1_size, dtype=REAL) + * _work = np.PyArray_DATA(work) + * if neu1 is None: # <<<<<<<<<<<<<< + * neu1 = zeros(model.layer1_size, dtype=REAL) + * _neu1 = np.PyArray_DATA(neu1) + */ + __pyx_t_9 = (__pyx_v_neu1 == Py_None); + __pyx_t_8 = (__pyx_t_9 != 0); + if (__pyx_t_8) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":626 + * _work = np.PyArray_DATA(work) + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) # <<<<<<<<<<<<<< + * _neu1 = np.PyArray_DATA(neu1) + * + */ + __pyx_t_12 = __Pyx_GetModuleGlobalName(__pyx_n_s_zeros); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_10); + __Pyx_GIVEREF(__pyx_t_10); + __pyx_t_10 = 0; + __pyx_t_10 = PyDict_New(); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_REAL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_t_10, __pyx_n_s_dtype, __pyx_t_4) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_12, __pyx_t_1, __pyx_t_10); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 626; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF_SET(__pyx_v_neu1, __pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L11; + } + __pyx_L11:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":627 + * if neu1 is None: + * neu1 = zeros(model.layer1_size, dtype=REAL) + * _neu1 = np.PyArray_DATA(neu1) # <<<<<<<<<<<<<< * + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) */ - if (!(likely(((__pyx_v__neu1) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__neu1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 816; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_neu1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__neu1))); + if (!(likely(((__pyx_v_neu1) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_neu1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 627; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v__neu1 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v_neu1))); - /* "trunk/gensim/models/doc2vec_inner.pyx":817 - * work = np.PyArray_DATA(_work) - * neu1 = np.PyArray_DATA(_neu1) - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":629 + * _neu1 = np.PyArray_DATA(neu1) * - * for i in range(sentence_len): + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) # <<<<<<<<<<<<<< + * j = 0 + * for i in range(document_len): */ - __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 817; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = 10000; - if (((__pyx_t_5 < __pyx_t_9) != 0)) { - __pyx_t_10 = __pyx_t_5; + __pyx_t_7 = PyObject_Length(__pyx_v_word_vocabs); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 629; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = 10000; + if (((__pyx_t_7 < __pyx_t_6) != 0)) { + __pyx_t_5 = __pyx_t_7; } else { - __pyx_t_10 = __pyx_t_9; + __pyx_t_5 = __pyx_t_6; } - __pyx_v_sentence_len = ((int)__pyx_t_10); + __pyx_v_document_len = ((int)__pyx_t_5); - /* "trunk/gensim/models/doc2vec_inner.pyx":819 - * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) + /* "trunk/gensim/models/doc2vec_inner.pyx":630 * - * for i in range(sentence_len): # <<<<<<<<<<<<<< - * word = sentence[i] + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * j = 0 # <<<<<<<<<<<<<< + * for i in range(document_len): + * word = word_vocabs[i] + */ + __pyx_v_j = 0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":631 + * document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + * j = 0 + * for i in range(document_len): # <<<<<<<<<<<<<< + * word = word_vocabs[i] * if word is None: */ - __pyx_t_2 = __pyx_v_sentence_len; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_i = __pyx_t_11; + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":820 - * - * for i in range(sentence_len): - * word = sentence[i] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":632 + * j = 0 + * for i in range(document_len): + * word = word_vocabs[i] # <<<<<<<<<<<<<< * if word is None: - * codelens[i] = 0 + * # shrink document to leave out word */ - __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 820; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - __Pyx_GOTREF(__pyx_t_6); - __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_6); - __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_word_vocabs, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_4 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 632; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_4); + __pyx_t_4 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":821 - * for i in range(sentence_len): - * word = sentence[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":633 + * for i in range(document_len): + * word = word_vocabs[i] * if word is None: # <<<<<<<<<<<<<< - * codelens[i] = 0 - * else: + * # shrink document to leave out word + * document_len = document_len - 1 */ - __pyx_t_4 = (__pyx_v_word == Py_None); - __pyx_t_12 = (__pyx_t_4 != 0); - if (__pyx_t_12) { + __pyx_t_8 = (__pyx_v_word == Py_None); + __pyx_t_9 = (__pyx_t_8 != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":822 - * word = sentence[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":635 * if word is None: - * codelens[i] = 0 # <<<<<<<<<<<<<< + * # shrink document to leave out word + * document_len = document_len - 1 # <<<<<<<<<<<<<< + * continue # leaving j unchanged + * else: + */ + __pyx_v_document_len = (__pyx_v_document_len - 1); + + /* "trunk/gensim/models/doc2vec_inner.pyx":636 + * # shrink document to leave out word + * document_len = document_len - 1 + * continue # leaving j unchanged # <<<<<<<<<<<<<< * else: - * indexes[i] = word.index + * indexes[j] = word.index */ - (__pyx_v_codelens[__pyx_v_i]) = 0; - goto __pyx_L7; + goto __pyx_L12_continue; } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":824 - * codelens[i] = 0 + /* "trunk/gensim/models/doc2vec_inner.pyx":638 + * continue # leaving j unchanged * else: - * indexes[i] = word.index # <<<<<<<<<<<<<< + * indexes[j] = word.index # <<<<<<<<<<<<<< * if hs: - * codelens[i] = len(word.code) + * codelens[j] = len(word.code) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 824; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 824; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - (__pyx_v_indexes[__pyx_v_i]) = __pyx_t_13; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 638; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_4); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 638; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + (__pyx_v_indexes[__pyx_v_j]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":825 + /* "trunk/gensim/models/doc2vec_inner.pyx":639 * else: - * indexes[i] = word.index + * indexes[j] = word.index * if hs: # <<<<<<<<<<<<<< - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) */ - __pyx_t_12 = (__pyx_v_hs != 0); - if (__pyx_t_12) { + __pyx_t_9 = (__pyx_v_hs != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":826 - * indexes[i] = word.index + /* "trunk/gensim/models/doc2vec_inner.pyx":640 + * indexes[j] = word.index * if hs: - * codelens[i] = len(word.code) # <<<<<<<<<<<<<< - * codes[i] = np.PyArray_DATA(word.code) - * points[i] = np.PyArray_DATA(word.point) + * codelens[j] = len(word.code) # <<<<<<<<<<<<<< + * codes[j] = np.PyArray_DATA(word.code) + * points[j] = np.PyArray_DATA(word.point) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_10 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - (__pyx_v_codelens[__pyx_v_i]) = ((int)__pyx_t_10); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 640; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyObject_Length(__pyx_t_4); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 640; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + (__pyx_v_codelens[__pyx_v_j]) = ((int)__pyx_t_5); - /* "trunk/gensim/models/doc2vec_inner.pyx":827 + /* "trunk/gensim/models/doc2vec_inner.pyx":641 * if hs: - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< - * points[i] = np.PyArray_DATA(word.point) + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< + * points[j] = np.PyArray_DATA(word.point) * else: */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 827; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 827; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_codes[__pyx_v_i]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 641; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 641; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_codes[__pyx_v_j]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_4))); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":828 - * codelens[i] = len(word.code) - * codes[i] = np.PyArray_DATA(word.code) - * points[i] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":642 + * codelens[j] = len(word.code) + * codes[j] = np.PyArray_DATA(word.code) + * points[j] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< * else: - * codelens[i] = 1 + * codelens[j] = 1 */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 828; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 828; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_points[__pyx_v_i]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - goto __pyx_L8; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 642; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 642; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + (__pyx_v_points[__pyx_v_j]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_4))); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L15; } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":830 - * points[i] = np.PyArray_DATA(word.point) + /* "trunk/gensim/models/doc2vec_inner.pyx":644 + * points[j] = np.PyArray_DATA(word.point) * else: - * codelens[i] = 1 # <<<<<<<<<<<<<< + * codelens[j] = 1 # <<<<<<<<<<<<<< * result += 1 - * # single randint() call avoids a big thread-sync slowdown + * j = j + 1 */ - (__pyx_v_codelens[__pyx_v_i]) = 1; + (__pyx_v_codelens[__pyx_v_j]) = 1; } - __pyx_L8:; + __pyx_L15:; - /* "trunk/gensim/models/doc2vec_inner.pyx":831 + /* "trunk/gensim/models/doc2vec_inner.pyx":645 * else: - * codelens[i] = 1 + * codelens[j] = 1 * result += 1 # <<<<<<<<<<<<<< - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): + * j = j + 1 + * */ __pyx_v_result = (__pyx_v_result + 1); - } - __pyx_L7:; - } - /* "trunk/gensim/models/doc2vec_inner.pyx":833 + /* "trunk/gensim/models/doc2vec_inner.pyx":646 + * codelens[j] = 1 * result += 1 - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< - * reduced_windows[i] = item + * j = j + 1 # <<<<<<<<<<<<<< * + * for i in range(doctag_len): */ - __pyx_t_2 = 0; - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_14); - __pyx_t_15 = NULL; - __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_15 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_15)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_15); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - __pyx_t_10 = 1; - } - } - __pyx_t_16 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_16); - if (__pyx_t_15) { - PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_15); __Pyx_GIVEREF(__pyx_t_15); __pyx_t_15 = NULL; - } - __Pyx_INCREF(__pyx_int_0); - PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_10, __pyx_int_0); - __Pyx_GIVEREF(__pyx_int_0); - PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_10, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_16, 2+__pyx_t_10, __pyx_t_14); - __Pyx_GIVEREF(__pyx_t_14); - __pyx_t_1 = 0; - __pyx_t_14 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (likely(PyList_CheckExact(__pyx_t_6)) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); __pyx_t_10 = 0; - __pyx_t_17 = NULL; - } else { - __pyx_t_10 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - for (;;) { - if (likely(!__pyx_t_17)) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_7)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - } else { - if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - } - } else { - __pyx_t_6 = __pyx_t_17(__pyx_t_7); - if (unlikely(!__pyx_t_6)) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 833; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - break; - } - __Pyx_GOTREF(__pyx_t_6); + __pyx_v_j = (__pyx_v_j + 1); } - __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_v_i = __pyx_t_2; - __pyx_t_2 = (__pyx_t_2 + 1); - - /* "trunk/gensim/models/doc2vec_inner.pyx":834 - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): - * reduced_windows[i] = item # <<<<<<<<<<<<<< - * - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) - */ - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 834; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_13; - - /* "trunk/gensim/models/doc2vec_inner.pyx":833 - * result += 1 - * # single randint() call avoids a big thread-sync slowdown - * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< - * reduced_windows[i] = item - * - */ + __pyx_L12_continue:; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":836 - * reduced_windows[i] = item + /* "trunk/gensim/models/doc2vec_inner.pyx":648 + * j = j + 1 * - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) # <<<<<<<<<<<<<< - * for i in range(lbl_length): - * word = lbls[i] + * for i in range(doctag_len): # <<<<<<<<<<<<<< + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 */ - __pyx_t_10 = PyObject_Length(__pyx_v_lbls); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 836; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = 10000; - if (((__pyx_t_10 < __pyx_t_9) != 0)) { - __pyx_t_5 = __pyx_t_10; - } else { - __pyx_t_5 = __pyx_t_9; - } - __pyx_v_lbl_length = ((int)__pyx_t_5); + __pyx_t_2 = __pyx_v_doctag_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; - /* "trunk/gensim/models/doc2vec_inner.pyx":837 + /* "trunk/gensim/models/doc2vec_inner.pyx":649 * - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) - * for i in range(lbl_length): # <<<<<<<<<<<<<< - * word = lbls[i] - * if word is None: - */ - __pyx_t_2 = __pyx_v_lbl_length; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_i = __pyx_t_11; - - /* "trunk/gensim/models/doc2vec_inner.pyx":838 - * lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) - * for i in range(lbl_length): - * word = lbls[i] # <<<<<<<<<<<<<< - * if word is None: - * lbl_codelens[i] = 0 - */ - __pyx_t_7 = __Pyx_GetItemInt(__pyx_v_lbls, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_7 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 838; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - __Pyx_GOTREF(__pyx_t_7); - __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_7); - __pyx_t_7 = 0; - - /* "trunk/gensim/models/doc2vec_inner.pyx":839 - * for i in range(lbl_length): - * word = lbls[i] - * if word is None: # <<<<<<<<<<<<<< - * lbl_codelens[i] = 0 - * else: - */ - __pyx_t_12 = (__pyx_v_word == Py_None); - __pyx_t_4 = (__pyx_t_12 != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":840 - * word = lbls[i] - * if word is None: - * lbl_codelens[i] = 0 # <<<<<<<<<<<<<< - * else: - * lbl_indexes[i] = word.index - */ - (__pyx_v_lbl_codelens[__pyx_v_i]) = 0; - goto __pyx_L13; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":842 - * lbl_codelens[i] = 0 - * else: - * lbl_indexes[i] = word.index # <<<<<<<<<<<<<< - * if hs: - * lbl_codelens[i] = len(word.code) - */ - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 842; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_7); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 842; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - (__pyx_v_lbl_indexes[__pyx_v_i]) = __pyx_t_13; - - /* "trunk/gensim/models/doc2vec_inner.pyx":843 - * else: - * lbl_indexes[i] = word.index - * if hs: # <<<<<<<<<<<<<< - * lbl_codelens[i] = len(word.code) - * else: - */ - __pyx_t_4 = (__pyx_v_hs != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/doc2vec_inner.pyx":844 - * lbl_indexes[i] = word.index - * if hs: - * lbl_codelens[i] = len(word.code) # <<<<<<<<<<<<<< - * else: - * lbl_codelens[i] = 1 - */ - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 844; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_5 = PyObject_Length(__pyx_t_7); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 844; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - (__pyx_v_lbl_codelens[__pyx_v_i]) = ((int)__pyx_t_5); - goto __pyx_L14; - } - /*else*/ { - - /* "trunk/gensim/models/doc2vec_inner.pyx":846 - * lbl_codelens[i] = len(word.code) - * else: - * lbl_codelens[i] = 1 # <<<<<<<<<<<<<< - * result += 1 + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] # <<<<<<<<<<<<<< + * result += 1 * */ - (__pyx_v_lbl_codelens[__pyx_v_i]) = 1; - } - __pyx_L14:; + __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_doctag_indexes, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_4 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 649; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_14 = __Pyx_PyInt_As_npy_uint32(__pyx_t_4); if (unlikely((__pyx_t_14 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 649; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + (__pyx_v__doctag_indexes[__pyx_v_i]) = __pyx_t_14; - /* "trunk/gensim/models/doc2vec_inner.pyx":847 - * else: - * lbl_codelens[i] = 1 - * result += 1 # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":650 + * for i in range(doctag_len): + * _doctag_indexes[i] = doctag_indexes[i] + * result += 1 # <<<<<<<<<<<<<< * - * # release GIL & train on the sentence + * # release GIL & train on the document */ - __pyx_v_result = (__pyx_v_result + 1); - } - __pyx_L13:; + __pyx_v_result = (__pyx_v_result + 1); } - /* "trunk/gensim/models/doc2vec_inner.pyx":850 + /* "trunk/gensim/models/doc2vec_inner.pyx":653 * - * # release GIL & train on the sentence + * # release GIL & train on the document * with nogil: # <<<<<<<<<<<<<< - * for i in range(sentence_len): - * if codelens[i] == 0: + * for i in range(document_len): + * j = i - window # negative OK: will pad with null word */ { #ifdef WITH_THREAD @@ -8001,205 +6814,354 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence #endif /*try:*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":851 - * # release GIL & train on the sentence + /* "trunk/gensim/models/doc2vec_inner.pyx":654 + * # release GIL & train on the document * with nogil: - * for i in range(sentence_len): # <<<<<<<<<<<<<< - * if codelens[i] == 0: - * continue + * for i in range(document_len): # <<<<<<<<<<<<<< + * j = i - window # negative OK: will pad with null word + * k = i + window + 1 # past document end OK: will pad with null word + */ + __pyx_t_2 = __pyx_v_document_len; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_2; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; + + /* "trunk/gensim/models/doc2vec_inner.pyx":655 + * with nogil: + * for i in range(document_len): + * j = i - window # negative OK: will pad with null word # <<<<<<<<<<<<<< + * k = i + window + 1 # past document end OK: will pad with null word + * + */ + __pyx_v_j = (__pyx_v_i - __pyx_v_window); + + /* "trunk/gensim/models/doc2vec_inner.pyx":656 + * for i in range(document_len): + * j = i - window # negative OK: will pad with null word + * k = i + window + 1 # past document end OK: will pad with null word # <<<<<<<<<<<<<< + * + * # compose l1 & clear work */ - __pyx_t_2 = __pyx_v_sentence_len; - for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { - __pyx_v_i = __pyx_t_11; + __pyx_v_k = ((__pyx_v_i + __pyx_v_window) + 1); - /* "trunk/gensim/models/doc2vec_inner.pyx":852 - * with nogil: - * for i in range(sentence_len): - * if codelens[i] == 0: # <<<<<<<<<<<<<< - * continue - * j = i - window + reduced_windows[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":659 + * + * # compose l1 & clear work + * for m in range(doctag_len): # <<<<<<<<<<<<<< + * # doc vector(s) + * memcpy(&_neu1[m * vector_size], &_doctag_vectors[_doctag_indexes[m] * vector_size], */ - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_i]) == 0) != 0); - if (__pyx_t_4) { + __pyx_t_15 = __pyx_v_doctag_len; + for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) { + __pyx_v_m = __pyx_t_16; - /* "trunk/gensim/models/doc2vec_inner.pyx":853 - * for i in range(sentence_len): - * if codelens[i] == 0: - * continue # <<<<<<<<<<<<<< - * j = i - window + reduced_windows[i] - * if j < 0: + /* "trunk/gensim/models/doc2vec_inner.pyx":661 + * for m in range(doctag_len): + * # doc vector(s) + * memcpy(&_neu1[m * vector_size], &_doctag_vectors[_doctag_indexes[m] * vector_size], # <<<<<<<<<<<<<< + * vector_size * cython.sizeof(REAL_t)) + * n = 0 */ - goto __pyx_L18_continue; + memcpy((&(__pyx_v__neu1[(__pyx_v_m * __pyx_v_vector_size)])), (&(__pyx_v__doctag_vectors[((__pyx_v__doctag_indexes[__pyx_v_m]) * __pyx_v_vector_size)])), (__pyx_v_vector_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); } - /* "trunk/gensim/models/doc2vec_inner.pyx":854 - * if codelens[i] == 0: - * continue - * j = i - window + reduced_windows[i] # <<<<<<<<<<<<<< - * if j < 0: - * j = 0 + /* "trunk/gensim/models/doc2vec_inner.pyx":663 + * memcpy(&_neu1[m * vector_size], &_doctag_vectors[_doctag_indexes[m] * vector_size], + * vector_size * cython.sizeof(REAL_t)) + * n = 0 # <<<<<<<<<<<<<< + * for m in range(j, k): + * # word vectors in window */ - __pyx_v_j = ((__pyx_v_i - __pyx_v_window) + (__pyx_v_reduced_windows[__pyx_v_i])); + __pyx_v_n = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":855 - * continue - * j = i - window + reduced_windows[i] - * if j < 0: # <<<<<<<<<<<<<< - * j = 0 - * k = i + window + 1 - reduced_windows[i] + /* "trunk/gensim/models/doc2vec_inner.pyx":664 + * vector_size * cython.sizeof(REAL_t)) + * n = 0 + * for m in range(j, k): # <<<<<<<<<<<<<< + * # word vectors in window + * if m == i: + */ + __pyx_t_15 = __pyx_v_k; + for (__pyx_t_16 = __pyx_v_j; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) { + __pyx_v_m = __pyx_t_16; + + /* "trunk/gensim/models/doc2vec_inner.pyx":666 + * for m in range(j, k): + * # word vectors in window + * if m == i: # <<<<<<<<<<<<<< + * continue + * if m < 0 or m >= document_len: */ - __pyx_t_4 = ((__pyx_v_j < 0) != 0); - if (__pyx_t_4) { + __pyx_t_9 = ((__pyx_v_m == __pyx_v_i) != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":856 - * j = i - window + reduced_windows[i] - * if j < 0: - * j = 0 # <<<<<<<<<<<<<< - * k = i + window + 1 - reduced_windows[i] - * if k > sentence_len: + /* "trunk/gensim/models/doc2vec_inner.pyx":667 + * # word vectors in window + * if m == i: + * continue # <<<<<<<<<<<<<< + * if m < 0 or m >= document_len: + * window_indexes[n] = null_word_index */ - __pyx_v_j = 0; - goto __pyx_L21; - } - __pyx_L21:; + goto __pyx_L25_continue; + } - /* "trunk/gensim/models/doc2vec_inner.pyx":857 - * if j < 0: - * j = 0 - * k = i + window + 1 - reduced_windows[i] # <<<<<<<<<<<<<< - * if k > sentence_len: - * k = sentence_len + /* "trunk/gensim/models/doc2vec_inner.pyx":668 + * if m == i: + * continue + * if m < 0 or m >= document_len: # <<<<<<<<<<<<<< + * window_indexes[n] = null_word_index + * else: */ - __pyx_v_k = (((__pyx_v_i + __pyx_v_window) + 1) - (__pyx_v_reduced_windows[__pyx_v_i])); + __pyx_t_8 = ((__pyx_v_m < 0) != 0); + if (!__pyx_t_8) { + } else { + __pyx_t_9 = __pyx_t_8; + goto __pyx_L29_bool_binop_done; + } + __pyx_t_8 = ((__pyx_v_m >= __pyx_v_document_len) != 0); + __pyx_t_9 = __pyx_t_8; + __pyx_L29_bool_binop_done:; + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":858 - * j = 0 - * k = i + window + 1 - reduced_windows[i] - * if k > sentence_len: # <<<<<<<<<<<<<< - * k = sentence_len - * if hs: + /* "trunk/gensim/models/doc2vec_inner.pyx":669 + * continue + * if m < 0 or m >= document_len: + * window_indexes[n] = null_word_index # <<<<<<<<<<<<<< + * else: + * window_indexes[n] = indexes[m] */ - __pyx_t_4 = ((__pyx_v_k > __pyx_v_sentence_len) != 0); - if (__pyx_t_4) { + (__pyx_v_window_indexes[__pyx_v_n]) = __pyx_v_null_word_index; + goto __pyx_L28; + } + /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":859 - * k = i + window + 1 - reduced_windows[i] - * if k > sentence_len: - * k = sentence_len # <<<<<<<<<<<<<< - * if hs: - * fast_sentence_dm_hs(points[i], codes[i], codelens, lbl_codelens, neu1, syn0, syn1, size, indexes, + /* "trunk/gensim/models/doc2vec_inner.pyx":671 + * window_indexes[n] = null_word_index + * else: + * window_indexes[n] = indexes[m] # <<<<<<<<<<<<<< + * n = n + 1 + * for m in range(2 * window): + */ + (__pyx_v_window_indexes[__pyx_v_n]) = (__pyx_v_indexes[__pyx_v_m]); + } + __pyx_L28:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":672 + * else: + * window_indexes[n] = indexes[m] + * n = n + 1 # <<<<<<<<<<<<<< + * for m in range(2 * window): + * memcpy(&_neu1[(doctag_len + m) * vector_size], &_word_vectors[window_indexes[m] * vector_size], */ - __pyx_v_k = __pyx_v_sentence_len; - goto __pyx_L22; + __pyx_v_n = (__pyx_v_n + 1); + __pyx_L25_continue:; + } + + /* "trunk/gensim/models/doc2vec_inner.pyx":673 + * window_indexes[n] = indexes[m] + * n = n + 1 + * for m in range(2 * window): # <<<<<<<<<<<<<< + * memcpy(&_neu1[(doctag_len + m) * vector_size], &_word_vectors[window_indexes[m] * vector_size], + * vector_size * cython.sizeof(REAL_t)) + */ + __pyx_t_6 = (2 * __pyx_v_window); + for (__pyx_t_15 = 0; __pyx_t_15 < __pyx_t_6; __pyx_t_15+=1) { + __pyx_v_m = __pyx_t_15; + + /* "trunk/gensim/models/doc2vec_inner.pyx":674 + * n = n + 1 + * for m in range(2 * window): + * memcpy(&_neu1[(doctag_len + m) * vector_size], &_word_vectors[window_indexes[m] * vector_size], # <<<<<<<<<<<<<< + * vector_size * cython.sizeof(REAL_t)) + * memset(_work, 0, layer1_size * cython.sizeof(REAL_t)) # work to accumulate l1 error + */ + memcpy((&(__pyx_v__neu1[((__pyx_v_doctag_len + __pyx_v_m) * __pyx_v_vector_size)])), (&(__pyx_v__word_vectors[((__pyx_v_window_indexes[__pyx_v_m]) * __pyx_v_vector_size)])), (__pyx_v_vector_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); } - __pyx_L22:; - /* "trunk/gensim/models/doc2vec_inner.pyx":860 - * if k > sentence_len: - * k = sentence_len + /* "trunk/gensim/models/doc2vec_inner.pyx":676 + * memcpy(&_neu1[(doctag_len + m) * vector_size], &_word_vectors[window_indexes[m] * vector_size], + * vector_size * cython.sizeof(REAL_t)) + * memset(_work, 0, layer1_size * cython.sizeof(REAL_t)) # work to accumulate l1 error # <<<<<<<<<<<<<< + * + * if hs: + */ + memset(__pyx_v__work, 0, (__pyx_v_layer1_size * (sizeof(__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)))); + + /* "trunk/gensim/models/doc2vec_inner.pyx":678 + * memset(_work, 0, layer1_size * cython.sizeof(REAL_t)) # work to accumulate l1 error + * * if hs: # <<<<<<<<<<<<<< - * fast_sentence_dm_hs(points[i], codes[i], codelens, lbl_codelens, neu1, syn0, syn1, size, indexes, - * lbl_indexes, _alpha, work, i, j, k, cbow_mean, lbl_length, tw, tl) + * fast_document_dmc_hs(points[i], codes[i], codelens[i], + * _neu1, syn1, _alpha, _work, */ - __pyx_t_4 = (__pyx_v_hs != 0); - if (__pyx_t_4) { + __pyx_t_9 = (__pyx_v_hs != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":861 - * k = sentence_len + /* "trunk/gensim/models/doc2vec_inner.pyx":679 + * * if hs: - * fast_sentence_dm_hs(points[i], codes[i], codelens, lbl_codelens, neu1, syn0, syn1, size, indexes, # <<<<<<<<<<<<<< - * lbl_indexes, _alpha, work, i, j, k, cbow_mean, lbl_length, tw, tl) - * if negative: + * fast_document_dmc_hs(points[i], codes[i], codelens[i], # <<<<<<<<<<<<<< + * _neu1, syn1, _alpha, _work, + * layer1_size, vector_size, _learn_hidden) */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), __pyx_v_codelens, __pyx_v_lbl_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, __pyx_v_indexes, __pyx_v_lbl_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean, __pyx_v_lbl_length, __pyx_v_tw, __pyx_v_tl); - goto __pyx_L23; + __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v__neu1, __pyx_v_syn1, __pyx_v__alpha, __pyx_v__work, __pyx_v_layer1_size, __pyx_v_vector_size, __pyx_v__learn_hidden); + goto __pyx_L33; } - __pyx_L23:; + __pyx_L33:; - /* "trunk/gensim/models/doc2vec_inner.pyx":863 - * fast_sentence_dm_hs(points[i], codes[i], codelens, lbl_codelens, neu1, syn0, syn1, size, indexes, - * lbl_indexes, _alpha, work, i, j, k, cbow_mean, lbl_length, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":682 + * _neu1, syn1, _alpha, _work, + * layer1_size, vector_size, _learn_hidden) * if negative: # <<<<<<<<<<<<<< - * next_random = fast_sentence_dm_neg(negative, table, table_len, codelens, lbl_codelens, neu1, syn0, - * syn1neg, size, indexes, lbl_indexes, _alpha, work, i, j, k, + * next_random = fast_document_dmc_neg(negative, table, table_len, next_random, + * _neu1, syn1neg, indexes[i], _alpha, _work, */ - __pyx_t_4 = (__pyx_v_negative != 0); - if (__pyx_t_4) { + __pyx_t_9 = (__pyx_v_negative != 0); + if (__pyx_t_9) { - /* "trunk/gensim/models/doc2vec_inner.pyx":864 - * lbl_indexes, _alpha, work, i, j, k, cbow_mean, lbl_length, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":683 + * layer1_size, vector_size, _learn_hidden) * if negative: - * next_random = fast_sentence_dm_neg(negative, table, table_len, codelens, lbl_codelens, neu1, syn0, # <<<<<<<<<<<<<< - * syn1neg, size, indexes, lbl_indexes, _alpha, work, i, j, k, - * cbow_mean, next_random, lbl_length, tw, tl) + * next_random = fast_document_dmc_neg(negative, table, table_len, next_random, # <<<<<<<<<<<<<< + * _neu1, syn1neg, indexes[i], _alpha, _work, + * layer1_size, vector_size, _learn_hidden) */ - __pyx_v_next_random = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_codelens, __pyx_v_lbl_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, __pyx_v_indexes, __pyx_v_lbl_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean, __pyx_v_next_random, __pyx_v_lbl_length, __pyx_v_tw, __pyx_v_tl); - goto __pyx_L24; + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_document_dmc_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_next_random, __pyx_v__neu1, __pyx_v_syn1neg, (__pyx_v_indexes[__pyx_v_i]), __pyx_v__alpha, __pyx_v__work, __pyx_v_layer1_size, __pyx_v_vector_size, __pyx_v__learn_hidden); + goto __pyx_L34; } - __pyx_L24:; - __pyx_L18_continue:; + __pyx_L34:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":687 + * layer1_size, vector_size, _learn_hidden) + * + * if _learn_doctags: # <<<<<<<<<<<<<< + * for m in range(doctag_len): + * our_saxpy(&vector_size, &_doctag_locks[_doctag_indexes[m]], &_work[m * vector_size], + */ + __pyx_t_9 = (__pyx_v__learn_doctags != 0); + if (__pyx_t_9) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":688 + * + * if _learn_doctags: + * for m in range(doctag_len): # <<<<<<<<<<<<<< + * our_saxpy(&vector_size, &_doctag_locks[_doctag_indexes[m]], &_work[m * vector_size], + * &ONE, &_doctag_vectors[_doctag_indexes[m] * vector_size], &ONE) + */ + __pyx_t_15 = __pyx_v_doctag_len; + for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) { + __pyx_v_m = __pyx_t_16; + + /* "trunk/gensim/models/doc2vec_inner.pyx":689 + * if _learn_doctags: + * for m in range(doctag_len): + * our_saxpy(&vector_size, &_doctag_locks[_doctag_indexes[m]], &_work[m * vector_size], # <<<<<<<<<<<<<< + * &ONE, &_doctag_vectors[_doctag_indexes[m] * vector_size], &ONE) + * if _learn_words: + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_vector_size), (&(__pyx_v__doctag_locks[(__pyx_v__doctag_indexes[__pyx_v_m])])), (&(__pyx_v__work[(__pyx_v_m * __pyx_v_vector_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v__doctag_vectors[((__pyx_v__doctag_indexes[__pyx_v_m]) * __pyx_v_vector_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + } + goto __pyx_L35; + } + __pyx_L35:; + + /* "trunk/gensim/models/doc2vec_inner.pyx":691 + * our_saxpy(&vector_size, &_doctag_locks[_doctag_indexes[m]], &_work[m * vector_size], + * &ONE, &_doctag_vectors[_doctag_indexes[m] * vector_size], &ONE) + * if _learn_words: # <<<<<<<<<<<<<< + * for m in range(2 * window): + * our_saxpy(&vector_size, &_word_locks[window_indexes[m]], &_work[(doctag_len + m) * vector_size], + */ + __pyx_t_9 = (__pyx_v__learn_words != 0); + if (__pyx_t_9) { + + /* "trunk/gensim/models/doc2vec_inner.pyx":692 + * &ONE, &_doctag_vectors[_doctag_indexes[m] * vector_size], &ONE) + * if _learn_words: + * for m in range(2 * window): # <<<<<<<<<<<<<< + * our_saxpy(&vector_size, &_word_locks[window_indexes[m]], &_work[(doctag_len + m) * vector_size], + * &ONE, &_word_vectors[window_indexes[m] * vector_size], &ONE) + */ + __pyx_t_6 = (2 * __pyx_v_window); + for (__pyx_t_15 = 0; __pyx_t_15 < __pyx_t_6; __pyx_t_15+=1) { + __pyx_v_m = __pyx_t_15; + + /* "trunk/gensim/models/doc2vec_inner.pyx":693 + * if _learn_words: + * for m in range(2 * window): + * our_saxpy(&vector_size, &_word_locks[window_indexes[m]], &_work[(doctag_len + m) * vector_size], # <<<<<<<<<<<<<< + * &ONE, &_word_vectors[window_indexes[m] * vector_size], &ONE) + * + */ + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy((&__pyx_v_vector_size), (&(__pyx_v__word_locks[(__pyx_v_window_indexes[__pyx_v_m])])), (&(__pyx_v__work[((__pyx_v_doctag_len + __pyx_v_m) * __pyx_v_vector_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), (&(__pyx_v__word_vectors[((__pyx_v_window_indexes[__pyx_v_m]) * __pyx_v_vector_size)])), (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); + } + goto __pyx_L38; + } + __pyx_L38:; } } - /* "trunk/gensim/models/doc2vec_inner.pyx":850 + /* "trunk/gensim/models/doc2vec_inner.pyx":653 * - * # release GIL & train on the sentence + * # release GIL & train on the document * with nogil: # <<<<<<<<<<<<<< - * for i in range(sentence_len): - * if codelens[i] == 0: + * for i in range(document_len): + * j = i - window # negative OK: will pad with null word */ /*finally:*/ { /*normal exit:*/{ #ifdef WITH_THREAD Py_BLOCK_THREADS #endif - goto __pyx_L17; + goto __pyx_L20; } - __pyx_L17:; + __pyx_L20:; } } - /* "trunk/gensim/models/doc2vec_inner.pyx":868 - * cbow_mean, next_random, lbl_length, tw, tl) + /* "trunk/gensim/models/doc2vec_inner.pyx":696 + * &ONE, &_word_vectors[window_indexes[m] * vector_size], &ONE) * * return result # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 868; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_t_4 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 696; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; goto __pyx_L0; - /* "trunk/gensim/models/doc2vec_inner.pyx":767 + /* "trunk/gensim/models/doc2vec_inner.pyx":551 * * - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_14); - __Pyx_XDECREF(__pyx_t_15); - __Pyx_XDECREF(__pyx_t_16); - __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_sentence_dm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("trunk.gensim.models.doc2vec_inner.train_document_dm_concat", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF(__pyx_v_word); - __Pyx_XDECREF(__pyx_v_item); + __Pyx_XDECREF(__pyx_v_work); + __Pyx_XDECREF(__pyx_v_neu1); + __Pyx_XDECREF(__pyx_v_word_vectors); + __Pyx_XDECREF(__pyx_v_word_locks); + __Pyx_XDECREF(__pyx_v_doctag_vectors); + __Pyx_XDECREF(__pyx_v_doctag_locks); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "trunk/gensim/models/doc2vec_inner.pyx":871 +/* "trunk/gensim/models/doc2vec_inner.pyx":699 * * * def init(): # <<<<<<<<<<<<<< @@ -8208,21 +7170,21 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_2train_sentence */ /* Python wrapper */ -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5init(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static char __pyx_doc_5trunk_6gensim_6models_13doc2vec_inner_4init[] = "\n Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized\n into table EXP_TABLE.\n\n "; -static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_5init = {"init", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5init, METH_NOARGS, __pyx_doc_5trunk_6gensim_6models_13doc2vec_inner_4init}; -static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_5init(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_7init(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static char __pyx_doc_5trunk_6gensim_6models_13doc2vec_inner_6init[] = "\n Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized\n into table EXP_TABLE.\n\n "; +static PyMethodDef __pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_7init = {"init", (PyCFunction)__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_7init, METH_NOARGS, __pyx_doc_5trunk_6gensim_6models_13doc2vec_inner_6init}; +static PyObject *__pyx_pw_5trunk_6gensim_6models_13doc2vec_inner_7init(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("init (wrapper)", 0); - __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(__pyx_self); + __pyx_r = __pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_6init(__pyx_self); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UNUSED PyObject *__pyx_self) { +static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_6init(CYTHON_UNUSED PyObject *__pyx_self) { int __pyx_v_i; float *__pyx_v_x; float *__pyx_v_y; @@ -8238,7 +7200,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN int __pyx_t_4; __Pyx_RefNannySetupContext("init", 0); - /* "trunk/gensim/models/doc2vec_inner.pyx":883 + /* "trunk/gensim/models/doc2vec_inner.pyx":709 * * cdef int i * cdef float *x = [10.0] # <<<<<<<<<<<<<< @@ -8248,7 +7210,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN __pyx_t_1[0] = ((float)10.0); __pyx_v_x = __pyx_t_1; - /* "trunk/gensim/models/doc2vec_inner.pyx":884 + /* "trunk/gensim/models/doc2vec_inner.pyx":710 * cdef int i * cdef float *x = [10.0] * cdef float *y = [0.01] # <<<<<<<<<<<<<< @@ -8258,7 +7220,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN __pyx_t_2[0] = ((float)0.01); __pyx_v_y = __pyx_t_2; - /* "trunk/gensim/models/doc2vec_inner.pyx":885 + /* "trunk/gensim/models/doc2vec_inner.pyx":711 * cdef float *x = [10.0] * cdef float *y = [0.01] * cdef float expected = 0.1 # <<<<<<<<<<<<<< @@ -8267,7 +7229,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN */ __pyx_v_expected = ((float)0.1); - /* "trunk/gensim/models/doc2vec_inner.pyx":886 + /* "trunk/gensim/models/doc2vec_inner.pyx":712 * cdef float *y = [0.01] * cdef float expected = 0.1 * cdef int size = 1 # <<<<<<<<<<<<<< @@ -8276,7 +7238,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN */ __pyx_v_size = 1; - /* "trunk/gensim/models/doc2vec_inner.pyx":891 + /* "trunk/gensim/models/doc2vec_inner.pyx":717 * * # build the sigmoid table * for i in range(EXP_TABLE_SIZE): # <<<<<<<<<<<<<< @@ -8286,7 +7248,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN for (__pyx_t_3 = 0; __pyx_t_3 < 1000; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "trunk/gensim/models/doc2vec_inner.pyx":892 + /* "trunk/gensim/models/doc2vec_inner.pyx":718 * # build the sigmoid table * for i in range(EXP_TABLE_SIZE): * EXP_TABLE[i] = exp((i / EXP_TABLE_SIZE * 2 - 1) * MAX_EXP) # <<<<<<<<<<<<<< @@ -8295,7 +7257,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN */ (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[__pyx_v_i]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)exp(((((__pyx_v_i / ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)1000)) * 2.0) - 1.0) * 6.0))); - /* "trunk/gensim/models/doc2vec_inner.pyx":893 + /* "trunk/gensim/models/doc2vec_inner.pyx":719 * for i in range(EXP_TABLE_SIZE): * EXP_TABLE[i] = exp((i / EXP_TABLE_SIZE * 2 - 1) * MAX_EXP) * EXP_TABLE[i] = (EXP_TABLE[i] / (EXP_TABLE[i] + 1)) # <<<<<<<<<<<<<< @@ -8305,7 +7267,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN (__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[__pyx_v_i]) = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)((__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[__pyx_v_i]) / ((__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_EXP_TABLE[__pyx_v_i]) + 1.0))); } - /* "trunk/gensim/models/doc2vec_inner.pyx":896 + /* "trunk/gensim/models/doc2vec_inner.pyx":722 * * # check whether sdot returns double or float * d_res = dsdot(&size, x, &ONE, y, &ONE) # <<<<<<<<<<<<<< @@ -8314,67 +7276,49 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN */ __pyx_v_d_res = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot((&__pyx_v_size), __pyx_v_x, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE), __pyx_v_y, (&__pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE)); - /* "trunk/gensim/models/doc2vec_inner.pyx":897 + /* "trunk/gensim/models/doc2vec_inner.pyx":723 * # check whether sdot returns double or float * d_res = dsdot(&size, x, &ONE, y, &ONE) * p_res = &d_res # <<<<<<<<<<<<<< * if (abs(d_res - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence0_dbow_hs + * our_dot = our_dot_double */ __pyx_v_p_res = ((float *)(&__pyx_v_d_res)); - /* "trunk/gensim/models/doc2vec_inner.pyx":898 + /* "trunk/gensim/models/doc2vec_inner.pyx":724 * d_res = dsdot(&size, x, &ONE, y, &ONE) * p_res = &d_res * if (abs(d_res - expected) < 0.0001): # <<<<<<<<<<<<<< - * fast_sentence_dbow_hs = fast_sentence0_dbow_hs - * fast_sentence_dbow_neg = fast_sentence0_dbow_neg + * our_dot = our_dot_double + * our_saxpy = saxpy */ __pyx_t_4 = ((fabs((__pyx_v_d_res - __pyx_v_expected)) < 0.0001) != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":899 + /* "trunk/gensim/models/doc2vec_inner.pyx":725 * p_res = &d_res * if (abs(d_res - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence0_dbow_hs # <<<<<<<<<<<<<< - * fast_sentence_dbow_neg = fast_sentence0_dbow_neg - * fast_sentence_dm_hs = fast_sentence0_dm_hs - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_hs; - - /* "trunk/gensim/models/doc2vec_inner.pyx":900 - * if (abs(d_res - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence0_dbow_hs - * fast_sentence_dbow_neg = fast_sentence0_dbow_neg # <<<<<<<<<<<<<< - * fast_sentence_dm_hs = fast_sentence0_dm_hs - * fast_sentence_dm_neg = fast_sentence0_dm_neg - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dbow_neg; - - /* "trunk/gensim/models/doc2vec_inner.pyx":901 - * fast_sentence_dbow_hs = fast_sentence0_dbow_hs - * fast_sentence_dbow_neg = fast_sentence0_dbow_neg - * fast_sentence_dm_hs = fast_sentence0_dm_hs # <<<<<<<<<<<<<< - * fast_sentence_dm_neg = fast_sentence0_dm_neg + * our_dot = our_dot_double # <<<<<<<<<<<<<< + * our_saxpy = saxpy * return 0 # double */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_hs; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_double; - /* "trunk/gensim/models/doc2vec_inner.pyx":902 - * fast_sentence_dbow_neg = fast_sentence0_dbow_neg - * fast_sentence_dm_hs = fast_sentence0_dm_hs - * fast_sentence_dm_neg = fast_sentence0_dm_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":726 + * if (abs(d_res - expected) < 0.0001): + * our_dot = our_dot_double + * our_saxpy = saxpy # <<<<<<<<<<<<<< * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence0_dm_neg; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy; - /* "trunk/gensim/models/doc2vec_inner.pyx":903 - * fast_sentence_dm_hs = fast_sentence0_dm_hs - * fast_sentence_dm_neg = fast_sentence0_dm_neg + /* "trunk/gensim/models/doc2vec_inner.pyx":727 + * our_dot = our_dot_double + * our_saxpy = saxpy * return 0 # double # <<<<<<<<<<<<<< * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence1_dbow_hs + * our_dot = our_dot_float */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_int_0); @@ -8382,55 +7326,37 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN goto __pyx_L0; } - /* "trunk/gensim/models/doc2vec_inner.pyx":904 - * fast_sentence_dm_neg = fast_sentence0_dm_neg + /* "trunk/gensim/models/doc2vec_inner.pyx":728 + * our_saxpy = saxpy * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): # <<<<<<<<<<<<<< - * fast_sentence_dbow_hs = fast_sentence1_dbow_hs - * fast_sentence_dbow_neg = fast_sentence1_dbow_neg + * our_dot = our_dot_float + * our_saxpy = saxpy */ __pyx_t_4 = ((fabsf(((__pyx_v_p_res[0]) - __pyx_v_expected)) < 0.0001) != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/doc2vec_inner.pyx":905 + /* "trunk/gensim/models/doc2vec_inner.pyx":729 * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence1_dbow_hs # <<<<<<<<<<<<<< - * fast_sentence_dbow_neg = fast_sentence1_dbow_neg - * fast_sentence_dm_hs = fast_sentence1_dm_hs - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_hs; - - /* "trunk/gensim/models/doc2vec_inner.pyx":906 - * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_dbow_hs = fast_sentence1_dbow_hs - * fast_sentence_dbow_neg = fast_sentence1_dbow_neg # <<<<<<<<<<<<<< - * fast_sentence_dm_hs = fast_sentence1_dm_hs - * fast_sentence_dm_neg = fast_sentence1_dm_neg - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dbow_neg; - - /* "trunk/gensim/models/doc2vec_inner.pyx":907 - * fast_sentence_dbow_hs = fast_sentence1_dbow_hs - * fast_sentence_dbow_neg = fast_sentence1_dbow_neg - * fast_sentence_dm_hs = fast_sentence1_dm_hs # <<<<<<<<<<<<<< - * fast_sentence_dm_neg = fast_sentence1_dm_neg + * our_dot = our_dot_float # <<<<<<<<<<<<<< + * our_saxpy = saxpy * return 1 # float */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_hs; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_float; - /* "trunk/gensim/models/doc2vec_inner.pyx":908 - * fast_sentence_dbow_neg = fast_sentence1_dbow_neg - * fast_sentence_dm_hs = fast_sentence1_dm_hs - * fast_sentence_dm_neg = fast_sentence1_dm_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":730 + * elif (abs(p_res[0] - expected) < 0.0001): + * our_dot = our_dot_float + * our_saxpy = saxpy # <<<<<<<<<<<<<< * return 1 # float * else: */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence1_dm_neg; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy = __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy; - /* "trunk/gensim/models/doc2vec_inner.pyx":909 - * fast_sentence_dm_hs = fast_sentence1_dm_hs - * fast_sentence_dm_neg = fast_sentence1_dm_neg + /* "trunk/gensim/models/doc2vec_inner.pyx":731 + * our_dot = our_dot_float + * our_saxpy = saxpy * return 1 # float # <<<<<<<<<<<<<< * else: * # neither => use cython loops, no BLAS @@ -8442,45 +7368,27 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN } /*else*/ { - /* "trunk/gensim/models/doc2vec_inner.pyx":913 + /* "trunk/gensim/models/doc2vec_inner.pyx":735 * # neither => use cython loops, no BLAS * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - * fast_sentence_dbow_hs = fast_sentence2_dbow_hs # <<<<<<<<<<<<<< - * fast_sentence_dbow_neg = fast_sentence2_dbow_neg - * fast_sentence_dm_hs = fast_sentence2_dm_hs - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_hs; - - /* "trunk/gensim/models/doc2vec_inner.pyx":914 - * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - * fast_sentence_dbow_hs = fast_sentence2_dbow_hs - * fast_sentence_dbow_neg = fast_sentence2_dbow_neg # <<<<<<<<<<<<<< - * fast_sentence_dm_hs = fast_sentence2_dm_hs - * fast_sentence_dm_neg = fast_sentence2_dm_neg - */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dbow_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dbow_neg; - - /* "trunk/gensim/models/doc2vec_inner.pyx":915 - * fast_sentence_dbow_hs = fast_sentence2_dbow_hs - * fast_sentence_dbow_neg = fast_sentence2_dbow_neg - * fast_sentence_dm_hs = fast_sentence2_dm_hs # <<<<<<<<<<<<<< - * fast_sentence_dm_neg = fast_sentence2_dm_neg + * our_dot = our_dot_noblas # <<<<<<<<<<<<<< + * our_saxpy = our_saxpy_noblas * return 2 */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_hs = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_hs; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_dot_noblas; - /* "trunk/gensim/models/doc2vec_inner.pyx":916 - * fast_sentence_dbow_neg = fast_sentence2_dbow_neg - * fast_sentence_dm_hs = fast_sentence2_dm_hs - * fast_sentence_dm_neg = fast_sentence2_dm_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/doc2vec_inner.pyx":736 + * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here + * our_dot = our_dot_noblas + * our_saxpy = our_saxpy_noblas # <<<<<<<<<<<<<< * return 2 * */ - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence_dm_neg = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_fast_sentence2_dm_neg; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy = __pyx_f_5trunk_6gensim_6models_13doc2vec_inner_our_saxpy_noblas; - /* "trunk/gensim/models/doc2vec_inner.pyx":917 - * fast_sentence_dm_hs = fast_sentence2_dm_hs - * fast_sentence_dm_neg = fast_sentence2_dm_neg + /* "trunk/gensim/models/doc2vec_inner.pyx":737 + * our_dot = our_dot_noblas + * our_saxpy = our_saxpy_noblas * return 2 # <<<<<<<<<<<<<< * * FAST_VERSION = init() # initialize the module @@ -8491,7 +7399,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_13doc2vec_inner_4init(CYTHON_UN goto __pyx_L0; } - /* "trunk/gensim/models/doc2vec_inner.pyx":871 + /* "trunk/gensim/models/doc2vec_inner.pyx":699 * * * def init(): # <<<<<<<<<<<<<< @@ -8662,7 +7570,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -8702,7 +7610,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P * * info.buf = PyArray_DATA(self) */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 222; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 222; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -8979,7 +7887,7 @@ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, P * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -9791,7 +8699,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx * * if ((child.byteorder == c'>' and little_endian) or */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__11, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -9843,7 +8751,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx * # One could encode it in the format string and have Cython * # complain instead, BUT: < and > in format strings also imply */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -9944,7 +8852,7 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx * * # Until ticket #99 is fixed, use integers to avoid warnings */ - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__13, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -10539,16 +9447,32 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_k_Users_scratch_Documents_dev2015, sizeof(__pyx_k_Users_scratch_Documents_dev2015), 0, 0, 1, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_kp_s__5, __pyx_k__5, sizeof(__pyx_k__5), 0, 0, 1, 0}, {&__pyx_n_s_alpha, __pyx_k_alpha, sizeof(__pyx_k_alpha), 0, 0, 1, 1}, {&__pyx_n_s_alpha_2, __pyx_k_alpha_2, sizeof(__pyx_k_alpha_2), 0, 0, 1, 1}, {&__pyx_n_s_cbow_mean, __pyx_k_cbow_mean, sizeof(__pyx_k_cbow_mean), 0, 0, 1, 1}, {&__pyx_n_s_code, __pyx_k_code, sizeof(__pyx_k_code), 0, 0, 1, 1}, {&__pyx_n_s_codelens, __pyx_k_codelens, sizeof(__pyx_k_codelens), 0, 0, 1, 1}, {&__pyx_n_s_codes, __pyx_k_codes, sizeof(__pyx_k_codes), 0, 0, 1, 1}, + {&__pyx_n_s_count, __pyx_k_count, sizeof(__pyx_k_count), 0, 0, 1, 1}, {&__pyx_n_s_cpointer, __pyx_k_cpointer, sizeof(__pyx_k_cpointer), 0, 0, 1, 1}, {&__pyx_n_s_d_res, __pyx_k_d_res, sizeof(__pyx_k_d_res), 0, 0, 1, 1}, + {&__pyx_n_s_dm_tag_count, __pyx_k_dm_tag_count, sizeof(__pyx_k_dm_tag_count), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_indexes, __pyx_k_doctag_indexes, sizeof(__pyx_k_doctag_indexes), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_indexes_2, __pyx_k_doctag_indexes_2, sizeof(__pyx_k_doctag_indexes_2), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_len, __pyx_k_doctag_len, sizeof(__pyx_k_doctag_len), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_locks, __pyx_k_doctag_locks, sizeof(__pyx_k_doctag_locks), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_locks_2, __pyx_k_doctag_locks_2, sizeof(__pyx_k_doctag_locks_2), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_syn0, __pyx_k_doctag_syn0, sizeof(__pyx_k_doctag_syn0), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_syn0_lockf, __pyx_k_doctag_syn0_lockf, sizeof(__pyx_k_doctag_syn0_lockf), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_vectors, __pyx_k_doctag_vectors, sizeof(__pyx_k_doctag_vectors), 0, 0, 1, 1}, + {&__pyx_n_s_doctag_vectors_2, __pyx_k_doctag_vectors_2, sizeof(__pyx_k_doctag_vectors_2), 0, 0, 1, 1}, + {&__pyx_n_s_document_len, __pyx_k_document_len, sizeof(__pyx_k_document_len), 0, 0, 1, 1}, + {&__pyx_n_s_docvecs, __pyx_k_docvecs, sizeof(__pyx_k_docvecs), 0, 0, 1, 1}, + {&__pyx_n_s_dtype, __pyx_k_dtype, sizeof(__pyx_k_dtype), 0, 0, 1, 1}, {&__pyx_n_s_enumerate, __pyx_k_enumerate, sizeof(__pyx_k_enumerate), 0, 0, 1, 1}, {&__pyx_n_s_expected, __pyx_k_expected, sizeof(__pyx_k_expected), 0, 0, 1, 1}, + {&__pyx_n_s_expected_doctag_len, __pyx_k_expected_doctag_len, sizeof(__pyx_k_expected_doctag_len), 0, 0, 1, 1}, {&__pyx_n_s_fblas, __pyx_k_fblas, sizeof(__pyx_k_fblas), 0, 0, 1, 1}, {&__pyx_n_s_float32, __pyx_k_float32, sizeof(__pyx_k_float32), 0, 0, 1, 1}, {&__pyx_n_s_hs, __pyx_k_hs, sizeof(__pyx_k_hs), 0, 0, 1, 1}, @@ -10557,18 +9481,21 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_index, __pyx_k_index, sizeof(__pyx_k_index), 0, 0, 1, 1}, {&__pyx_n_s_indexes, __pyx_k_indexes, sizeof(__pyx_k_indexes), 0, 0, 1, 1}, {&__pyx_n_s_init, __pyx_k_init, sizeof(__pyx_k_init), 0, 0, 1, 1}, + {&__pyx_n_s_inv_count, __pyx_k_inv_count, sizeof(__pyx_k_inv_count), 0, 0, 1, 1}, {&__pyx_n_s_item, __pyx_k_item, sizeof(__pyx_k_item), 0, 0, 1, 1}, {&__pyx_n_s_j, __pyx_k_j, sizeof(__pyx_k_j), 0, 0, 1, 1}, {&__pyx_n_s_k, __pyx_k_k, sizeof(__pyx_k_k), 0, 0, 1, 1}, {&__pyx_n_s_layer1_size, __pyx_k_layer1_size, sizeof(__pyx_k_layer1_size), 0, 0, 1, 1}, - {&__pyx_n_s_lbl_codelens, __pyx_k_lbl_codelens, sizeof(__pyx_k_lbl_codelens), 0, 0, 1, 1}, - {&__pyx_n_s_lbl_codes, __pyx_k_lbl_codes, sizeof(__pyx_k_lbl_codes), 0, 0, 1, 1}, - {&__pyx_n_s_lbl_indexes, __pyx_k_lbl_indexes, sizeof(__pyx_k_lbl_indexes), 0, 0, 1, 1}, - {&__pyx_n_s_lbl_length, __pyx_k_lbl_length, sizeof(__pyx_k_lbl_length), 0, 0, 1, 1}, - {&__pyx_n_s_lbl_points, __pyx_k_lbl_points, sizeof(__pyx_k_lbl_points), 0, 0, 1, 1}, - {&__pyx_n_s_lbls, __pyx_k_lbls, sizeof(__pyx_k_lbls), 0, 0, 1, 1}, + {&__pyx_n_s_learn_doctags, __pyx_k_learn_doctags, sizeof(__pyx_k_learn_doctags), 0, 0, 1, 1}, + {&__pyx_n_s_learn_doctags_2, __pyx_k_learn_doctags_2, sizeof(__pyx_k_learn_doctags_2), 0, 0, 1, 1}, + {&__pyx_n_s_learn_hidden, __pyx_k_learn_hidden, sizeof(__pyx_k_learn_hidden), 0, 0, 1, 1}, + {&__pyx_n_s_learn_hidden_2, __pyx_k_learn_hidden_2, sizeof(__pyx_k_learn_hidden_2), 0, 0, 1, 1}, + {&__pyx_n_s_learn_words, __pyx_k_learn_words, sizeof(__pyx_k_learn_words), 0, 0, 1, 1}, + {&__pyx_n_s_learn_words_2, __pyx_k_learn_words_2, sizeof(__pyx_k_learn_words_2), 0, 0, 1, 1}, + {&__pyx_n_s_m, __pyx_k_m, sizeof(__pyx_k_m), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_model, __pyx_k_model, sizeof(__pyx_k_model), 0, 0, 1, 1}, + {&__pyx_n_s_n, __pyx_k_n, sizeof(__pyx_k_n), 0, 0, 1, 1}, {&__pyx_kp_u_ndarray_is_not_C_contiguous, __pyx_k_ndarray_is_not_C_contiguous, sizeof(__pyx_k_ndarray_is_not_C_contiguous), 0, 1, 0, 0}, {&__pyx_kp_u_ndarray_is_not_Fortran_contiguou, __pyx_k_ndarray_is_not_Fortran_contiguou, sizeof(__pyx_k_ndarray_is_not_Fortran_contiguou), 0, 1, 0, 0}, {&__pyx_n_s_negative, __pyx_k_negative, sizeof(__pyx_k_negative), 0, 0, 1, 1}, @@ -10576,10 +9503,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_neu1_2, __pyx_k_neu1_2, sizeof(__pyx_k_neu1_2), 0, 0, 1, 1}, {&__pyx_n_s_next_random, __pyx_k_next_random, sizeof(__pyx_k_next_random), 0, 0, 1, 1}, {&__pyx_n_s_np, __pyx_k_np, sizeof(__pyx_k_np), 0, 0, 1, 1}, + {&__pyx_n_s_null_word_index, __pyx_k_null_word_index, sizeof(__pyx_k_null_word_index), 0, 0, 1, 1}, {&__pyx_n_s_numpy, __pyx_k_numpy, sizeof(__pyx_k_numpy), 0, 0, 1, 1}, {&__pyx_n_s_p_res, __pyx_k_p_res, sizeof(__pyx_k_p_res), 0, 0, 1, 1}, {&__pyx_n_s_point, __pyx_k_point, sizeof(__pyx_k_point), 0, 0, 1, 1}, {&__pyx_n_s_points, __pyx_k_points, sizeof(__pyx_k_points), 0, 0, 1, 1}, + {&__pyx_n_s_predict_word, __pyx_k_predict_word, sizeof(__pyx_k_predict_word), 0, 0, 1, 1}, {&__pyx_n_s_randint, __pyx_k_randint, sizeof(__pyx_k_randint), 0, 0, 1, 1}, {&__pyx_n_s_random, __pyx_k_random, sizeof(__pyx_k_random), 0, 0, 1, 1}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, @@ -10589,36 +9518,43 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_scipy_linalg_blas, __pyx_k_scipy_linalg_blas, sizeof(__pyx_k_scipy_linalg_blas), 0, 0, 1, 1}, {&__pyx_n_s_scopy, __pyx_k_scopy, sizeof(__pyx_k_scopy), 0, 0, 1, 1}, {&__pyx_n_s_sdot, __pyx_k_sdot, sizeof(__pyx_k_sdot), 0, 0, 1, 1}, - {&__pyx_n_s_sentence, __pyx_k_sentence, sizeof(__pyx_k_sentence), 0, 0, 1, 1}, - {&__pyx_n_s_sentence_len, __pyx_k_sentence_len, sizeof(__pyx_k_sentence_len), 0, 0, 1, 1}, {&__pyx_n_s_size, __pyx_k_size, sizeof(__pyx_k_size), 0, 0, 1, 1}, {&__pyx_n_s_snrm2, __pyx_k_snrm2, sizeof(__pyx_k_snrm2), 0, 0, 1, 1}, {&__pyx_n_s_sscal, __pyx_k_sscal, sizeof(__pyx_k_sscal), 0, 0, 1, 1}, {&__pyx_n_s_syn0, __pyx_k_syn0, sizeof(__pyx_k_syn0), 0, 0, 1, 1}, + {&__pyx_n_s_syn0_lockf, __pyx_k_syn0_lockf, sizeof(__pyx_k_syn0_lockf), 0, 0, 1, 1}, {&__pyx_n_s_syn1, __pyx_k_syn1, sizeof(__pyx_k_syn1), 0, 0, 1, 1}, {&__pyx_n_s_syn1neg, __pyx_k_syn1neg, sizeof(__pyx_k_syn1neg), 0, 0, 1, 1}, {&__pyx_n_s_table, __pyx_k_table, sizeof(__pyx_k_table), 0, 0, 1, 1}, {&__pyx_n_s_table_len, __pyx_k_table_len, sizeof(__pyx_k_table_len), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_tl, __pyx_k_tl, sizeof(__pyx_k_tl), 0, 0, 1, 1}, - {&__pyx_n_s_train_lbls, __pyx_k_train_lbls, sizeof(__pyx_k_train_lbls), 0, 0, 1, 1}, - {&__pyx_n_s_train_sentence_dbow, __pyx_k_train_sentence_dbow, sizeof(__pyx_k_train_sentence_dbow), 0, 0, 1, 1}, - {&__pyx_n_s_train_sentence_dm, __pyx_k_train_sentence_dm, sizeof(__pyx_k_train_sentence_dm), 0, 0, 1, 1}, + {&__pyx_n_s_train_document_dbow, __pyx_k_train_document_dbow, sizeof(__pyx_k_train_document_dbow), 0, 0, 1, 1}, + {&__pyx_n_s_train_document_dm, __pyx_k_train_document_dm, sizeof(__pyx_k_train_document_dm), 0, 0, 1, 1}, + {&__pyx_n_s_train_document_dm_concat, __pyx_k_train_document_dm_concat, sizeof(__pyx_k_train_document_dm_concat), 0, 0, 1, 1}, {&__pyx_n_s_train_words, __pyx_k_train_words, sizeof(__pyx_k_train_words), 0, 0, 1, 1}, + {&__pyx_n_s_train_words_2, __pyx_k_train_words_2, sizeof(__pyx_k_train_words_2), 0, 0, 1, 1}, {&__pyx_n_s_trunk_gensim_models_doc2vec_inne, __pyx_k_trunk_gensim_models_doc2vec_inne, sizeof(__pyx_k_trunk_gensim_models_doc2vec_inne), 0, 0, 1, 1}, - {&__pyx_n_s_tw, __pyx_k_tw, sizeof(__pyx_k_tw), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, + {&__pyx_n_s_vector_size, __pyx_k_vector_size, sizeof(__pyx_k_vector_size), 0, 0, 1, 1}, + {&__pyx_n_s_vocab, __pyx_k_vocab, sizeof(__pyx_k_vocab), 0, 0, 1, 1}, {&__pyx_n_s_window, __pyx_k_window, sizeof(__pyx_k_window), 0, 0, 1, 1}, + {&__pyx_n_s_window_indexes, __pyx_k_window_indexes, sizeof(__pyx_k_window_indexes), 0, 0, 1, 1}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, + {&__pyx_n_s_word_locks, __pyx_k_word_locks, sizeof(__pyx_k_word_locks), 0, 0, 1, 1}, + {&__pyx_n_s_word_locks_2, __pyx_k_word_locks_2, sizeof(__pyx_k_word_locks_2), 0, 0, 1, 1}, + {&__pyx_n_s_word_vectors, __pyx_k_word_vectors, sizeof(__pyx_k_word_vectors), 0, 0, 1, 1}, + {&__pyx_n_s_word_vectors_2, __pyx_k_word_vectors_2, sizeof(__pyx_k_word_vectors_2), 0, 0, 1, 1}, + {&__pyx_n_s_word_vocabs, __pyx_k_word_vocabs, sizeof(__pyx_k_word_vocabs), 0, 0, 1, 1}, {&__pyx_n_s_work, __pyx_k_work, sizeof(__pyx_k_work), 0, 0, 1, 1}, {&__pyx_n_s_work_2, __pyx_k_work_2, sizeof(__pyx_k_work_2), 0, 0, 1, 1}, {&__pyx_n_s_x, __pyx_k_x, sizeof(__pyx_k_x), 0, 0, 1, 1}, {&__pyx_n_s_y, __pyx_k_y, sizeof(__pyx_k_y), 0, 0, 1, 1}, + {&__pyx_n_s_zeros, __pyx_k_zeros, sizeof(__pyx_k_zeros), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 737; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 355; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; @@ -10630,34 +9566,48 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "trunk/gensim/models/doc2vec_inner.pyx":716 + /* "trunk/gensim/models/doc2vec_inner.pyx":329 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_tuple_ = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple_)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple_ = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple_)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - __pyx_tuple__2 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 716; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__2 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "trunk/gensim/models/doc2vec_inner.pyx":812 + /* "trunk/gensim/models/doc2vec_inner.pyx":462 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_tuple__3 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__3 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - __pyx_tuple__4 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 812; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__4 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 462; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); + /* "trunk/gensim/models/doc2vec_inner.pyx":619 + * table = (np.PyArray_DATA(model.table)) + * table_len = len(model.table) + * next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # <<<<<<<<<<<<<< + * + * # convert Python structures to primitive types, so we can release the GIL + */ + __pyx_tuple__6 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_tuple__7 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); + /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":218 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): @@ -10665,9 +9615,9 @@ static int __Pyx_InitCachedConstants(void) { * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_C_contiguous); if (unlikely(!__pyx_tuple__5)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); + __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_C_contiguous); if (unlikely(!__pyx_tuple__8)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":222 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) @@ -10676,9 +9626,9 @@ static int __Pyx_InitCachedConstants(void) { * * info.buf = PyArray_DATA(self) */ - __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_Fortran_contiguou); if (unlikely(!__pyx_tuple__6)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 222; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_Fortran_contiguou); if (unlikely(!__pyx_tuple__9)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 222; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":260 * if ((descr.byteorder == c'>' and little_endian) or @@ -10687,9 +9637,9 @@ static int __Pyx_InitCachedConstants(void) { * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" */ - __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__7)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__10)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":802 * @@ -10698,9 +9648,9 @@ static int __Pyx_InitCachedConstants(void) { * * if ((child.byteorder == c'>' and little_endian) or */ - __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor); if (unlikely(!__pyx_tuple__8)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); + __pyx_tuple__11 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor); if (unlikely(!__pyx_tuple__11)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":806 * if ((child.byteorder == c'>' and little_endian) or @@ -10709,9 +9659,9 @@ static int __Pyx_InitCachedConstants(void) { * # One could encode it in the format string and have Cython * # complain instead, BUT: < and > in format strings also imply */ - __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__9)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); + __pyx_tuple__12 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__12)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 806; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":826 * t = child.type_num @@ -10720,45 +9670,57 @@ static int __Pyx_InitCachedConstants(void) { * * # Until ticket #99 is fixed, use integers to avoid warnings */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor_2); if (unlikely(!__pyx_tuple__10)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); + __pyx_tuple__13 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor_2); if (unlikely(!__pyx_tuple__13)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 826; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); - /* "trunk/gensim/models/doc2vec_inner.pyx":675 - * return next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":268 * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, # <<<<<<<<<<<<<< + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - __pyx_tuple__11 = PyTuple_Pack(35, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_lbls, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_train_words, __pyx_n_s_train_lbls, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_tw, __pyx_n_s_tl, __pyx_n_s_syn0, __pyx_n_s_work_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_lbl_codelens, __pyx_n_s_indexes, __pyx_n_s_lbl_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_lbl_length, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__11); - __Pyx_GIVEREF(__pyx_tuple__11); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(7, 0, 35, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_dbow, 675, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__14 = PyTuple_Pack(46, __pyx_n_s_model, __pyx_n_s_word_vocabs, __pyx_n_s_doctag_indexes, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_train_words, __pyx_n_s_learn_doctags, __pyx_n_s_learn_words, __pyx_n_s_learn_hidden, __pyx_n_s_word_vectors, __pyx_n_s_word_locks, __pyx_n_s_doctag_vectors, __pyx_n_s_doctag_locks, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_train_words_2, __pyx_n_s_learn_words_2, __pyx_n_s_learn_hidden_2, __pyx_n_s_learn_doctags_2, __pyx_n_s_word_vectors_2, __pyx_n_s_doctag_vectors_2, __pyx_n_s_word_locks_2, __pyx_n_s_doctag_locks_2, __pyx_n_s_work_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_doctag_indexes_2, __pyx_n_s_reduced_windows, __pyx_n_s_document_len, __pyx_n_s_doctag_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_predict_word, __pyx_n_s_item, __pyx_n_s_k); if (unlikely(!__pyx_tuple__14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__14); + __Pyx_GIVEREF(__pyx_tuple__14); + __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(13, 0, 46, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_document_dbow, 268, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "trunk/gensim/models/doc2vec_inner.pyx":767 + /* "trunk/gensim/models/doc2vec_inner.pyx":399 * * - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - __pyx_tuple__13 = PyTuple_Pack(41, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_lbls, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_neu1, __pyx_n_s_train_words, __pyx_n_s_train_lbls, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_tw, __pyx_n_s_tl, __pyx_n_s_cbow_mean, __pyx_n_s_syn0, __pyx_n_s_work_2, __pyx_n_s_neu1_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_lbl_codelens, __pyx_n_s_indexes, __pyx_n_s_lbl_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_lbl_length, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_lbl_points, __pyx_n_s_lbl_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__13)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__13); - __Pyx_GIVEREF(__pyx_tuple__13); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(8, 0, 41, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_dm, 767, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__16 = PyTuple_Pack(50, __pyx_n_s_model, __pyx_n_s_word_vocabs, __pyx_n_s_doctag_indexes, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_neu1, __pyx_n_s_learn_doctags, __pyx_n_s_learn_words, __pyx_n_s_learn_hidden, __pyx_n_s_word_vectors, __pyx_n_s_word_locks, __pyx_n_s_doctag_vectors, __pyx_n_s_doctag_locks, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_learn_doctags_2, __pyx_n_s_learn_words_2, __pyx_n_s_learn_hidden_2, __pyx_n_s_cbow_mean, __pyx_n_s_count, __pyx_n_s_inv_count, __pyx_n_s_word_vectors_2, __pyx_n_s_doctag_vectors_2, __pyx_n_s_word_locks_2, __pyx_n_s_doctag_locks_2, __pyx_n_s_work_2, __pyx_n_s_neu1_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_doctag_indexes_2, __pyx_n_s_reduced_windows, __pyx_n_s_document_len, __pyx_n_s_doctag_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_m, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + __pyx_codeobj__17 = (PyObject*)__Pyx_PyCode_New(13, 0, 50, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_document_dm, 399, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "trunk/gensim/models/doc2vec_inner.pyx":871 + /* "trunk/gensim/models/doc2vec_inner.pyx":551 + * + * + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + */ + __pyx_tuple__18 = PyTuple_Pack(50, __pyx_n_s_model, __pyx_n_s_word_vocabs, __pyx_n_s_doctag_indexes, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_neu1, __pyx_n_s_learn_doctags, __pyx_n_s_learn_words, __pyx_n_s_learn_hidden, __pyx_n_s_word_vectors, __pyx_n_s_word_locks, __pyx_n_s_doctag_vectors, __pyx_n_s_doctag_locks, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_learn_doctags_2, __pyx_n_s_learn_words_2, __pyx_n_s_learn_hidden_2, __pyx_n_s_word_vectors_2, __pyx_n_s_doctag_vectors_2, __pyx_n_s_word_locks_2, __pyx_n_s_doctag_locks_2, __pyx_n_s_work_2, __pyx_n_s_neu1_2, __pyx_n_s_alpha_2, __pyx_n_s_layer1_size, __pyx_n_s_vector_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_doctag_indexes_2, __pyx_n_s_window_indexes, __pyx_n_s_document_len, __pyx_n_s_doctag_len, __pyx_n_s_window, __pyx_n_s_expected_doctag_len, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_m, __pyx_n_s_n, __pyx_n_s_result, __pyx_n_s_null_word_index, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word); if (unlikely(!__pyx_tuple__18)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__18); + __Pyx_GIVEREF(__pyx_tuple__18); + __pyx_codeobj__19 = (PyObject*)__Pyx_PyCode_New(13, 0, 50, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__18, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_document_dm_concat, 551, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__19)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + + /* "trunk/gensim/models/doc2vec_inner.pyx":699 * * * def init(): # <<<<<<<<<<<<<< * """ * Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized */ - __pyx_tuple__15 = PyTuple_Pack(7, __pyx_n_s_i, __pyx_n_s_x, __pyx_n_s_y, __pyx_n_s_expected, __pyx_n_s_size, __pyx_n_s_d_res, __pyx_n_s_p_res); if (unlikely(!__pyx_tuple__15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 871; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__15); - __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__16 = (PyObject*)__Pyx_PyCode_New(0, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_init, 871, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 871; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__20 = PyTuple_Pack(7, __pyx_n_s_i, __pyx_n_s_x, __pyx_n_s_y, __pyx_n_s_expected, __pyx_n_s_size, __pyx_n_s_d_res, __pyx_n_s_p_res); if (unlikely(!__pyx_tuple__20)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_tuple__20); + __Pyx_GIVEREF(__pyx_tuple__20); + __pyx_codeobj__21 = (PyObject*)__Pyx_PyCode_New(0, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__20, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_init, 699, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__21)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -10880,159 +9842,187 @@ PyMODINIT_FUNC PyInit_doc2vec_inner(void) * * import cython * import numpy as np # <<<<<<<<<<<<<< + * from numpy import zeros, float32 as REAL * cimport numpy as np - * */ __pyx_t_1 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":20 + /* "trunk/gensim/models/doc2vec_inner.pyx":12 + * import cython + * import numpy as np + * from numpy import zeros, float32 as REAL # <<<<<<<<<<<<<< + * cimport numpy as np + * + */ + __pyx_t_1 = PyList_New(2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_zeros); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_zeros); + __Pyx_GIVEREF(__pyx_n_s_zeros); + __Pyx_INCREF(__pyx_n_s_float32); + PyList_SET_ITEM(__pyx_t_1, 1, __pyx_n_s_float32); + __Pyx_GIVEREF(__pyx_n_s_float32); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_numpy, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_zeros); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_zeros, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_float32); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_REAL, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":21 * void* PyCObject_AsVoidPtr(object obj) * * from scipy.linalg.blas import fblas # <<<<<<<<<<<<<< * * REAL = np.float32 */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_fblas); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_fblas); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_fblas); __Pyx_GIVEREF(__pyx_n_s_fblas); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_scipy_linalg_blas, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_Import(__pyx_n_s_scipy_linalg_blas, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_fblas, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_fblas, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":22 + /* "trunk/gensim/models/doc2vec_inner.pyx":23 * from scipy.linalg.blas import fblas * * REAL = np.float32 # <<<<<<<<<<<<<< * ctypedef np.float32_t REAL_t * */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 22; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_float32); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 22; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem(__pyx_d, __pyx_n_s_REAL, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 22; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_float32); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (PyDict_SetItem(__pyx_d, __pyx_n_s_REAL, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":57 - * int i, int j, int k, int cbow_mean, unsigned long long next_random, int lbl_length, int tw, int tl) nogil + /* "trunk/gensim/models/doc2vec_inner.pyx":35 + * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil * * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x # <<<<<<<<<<<<<< * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_scopy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_scopy); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_scopy = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_scopy_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_scopy = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_scopy_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":58 + /* "trunk/gensim/models/doc2vec_inner.pyx":36 * * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x # <<<<<<<<<<<<<< * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_saxpy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_saxpy); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_saxpy_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_saxpy = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_saxpy_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":59 + /* "trunk/gensim/models/doc2vec_inner.pyx":37 * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) # <<<<<<<<<<<<<< * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_sdot); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sdot_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sdot = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sdot_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":60 + /* "trunk/gensim/models/doc2vec_inner.pyx":38 * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) # <<<<<<<<<<<<<< * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_sdot); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_dsdot_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_dsdot = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_dsdot_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":61 + /* "trunk/gensim/models/doc2vec_inner.pyx":39 * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) # <<<<<<<<<<<<<< * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x - * cdef fast_sentence_dbow_hs_ptr fast_sentence_dbow_hs + * */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_snrm2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_snrm2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_snrm2 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_snrm2_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_snrm2 = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_snrm2_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":62 + /* "trunk/gensim/models/doc2vec_inner.pyx":40 * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x # <<<<<<<<<<<<<< - * cdef fast_sentence_dbow_hs_ptr fast_sentence_dbow_hs - * cdef fast_sentence_dbow_neg_ptr fast_sentence_dbow_neg + * + * DEF EXP_TABLE_SIZE = 1000 */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sscal); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_sscal); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sscal_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_sscal = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_sscal_ptr)PyCObject_AsVoidPtr(__pyx_t_2)); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":73 + /* "trunk/gensim/models/doc2vec_inner.pyx":47 * cdef REAL_t[EXP_TABLE_SIZE] EXP_TABLE * * cdef int ONE = 1 # <<<<<<<<<<<<<< @@ -11041,88 +10031,100 @@ PyMODINIT_FUNC PyInit_doc2vec_inner(void) */ __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONE = 1; - /* "trunk/gensim/models/doc2vec_inner.pyx":74 + /* "trunk/gensim/models/doc2vec_inner.pyx":48 * * cdef int ONE = 1 * cdef REAL_t ONEF = 1.0 # <<<<<<<<<<<<<< * - * cdef void fast_sentence0_dbow_hs( + * # function implementations swapped based on BLAS detected */ __pyx_v_5trunk_6gensim_6models_13doc2vec_inner_ONEF = ((__pyx_t_5trunk_6gensim_6models_13doc2vec_inner_REAL_t)1.0); - /* "trunk/gensim/models/doc2vec_inner.pyx":675 - * return next_random + /* "trunk/gensim/models/doc2vec_inner.pyx":268 * - * def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * + * def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, # <<<<<<<<<<<<<< + * train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_1train_sentence_dbow, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_dbow, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 675; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_1train_document_dbow, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_document_dbow, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 268; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":767 + /* "trunk/gensim/models/doc2vec_inner.pyx":399 * * - * def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): # <<<<<<<<<<<<<< - * cdef int hs = model.hs - * cdef int negative = model.negative + * def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_3train_sentence_dm, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_dm, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 767; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_3train_document_dm, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_document_dm, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 399; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "trunk/gensim/models/doc2vec_inner.pyx":551 + * + * + * def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, # <<<<<<<<<<<<<< + * learn_doctags=True, learn_words=True, learn_hidden=True, + * word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + */ + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_5train_document_dm_concat, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_document_dm_concat, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 551; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":871 + /* "trunk/gensim/models/doc2vec_inner.pyx":699 * * * def init(): # <<<<<<<<<<<<<< * """ * Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_5init, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 871; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_init, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 871; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_13doc2vec_inner_7init, NULL, __pyx_n_s_trunk_gensim_models_doc2vec_inne); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_init, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "trunk/gensim/models/doc2vec_inner.pyx":919 + /* "trunk/gensim/models/doc2vec_inner.pyx":739 * return 2 * * FAST_VERSION = init() # initialize the module # <<<<<<<<<<<<<< */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_init); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 919; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_init); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 739; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_1); if (likely(__pyx_t_3)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); + __Pyx_DECREF_SET(__pyx_t_1, function); } } if (__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 919; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 739; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 919; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 739; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FAST_VERSION, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 919; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FAST_VERSION, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 739; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "trunk/gensim/models/doc2vec_inner.pyx":1 * #!/usr/bin/env cython # <<<<<<<<<<<<<< * # cython: boundscheck=False * # cython: wraparound=False */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "../../../../../../miniconda3/envs/gensim_cenv/lib/python3.4/site-packages/Cython/Includes/numpy/__init__.pxd":979 * arr.base = baseptr diff --git a/gensim/models/doc2vec_inner.pyx b/gensim/models/doc2vec_inner.pyx index 5fb564f7e9..af8c77aee9 100644 --- a/gensim/models/doc2vec_inner.pyx +++ b/gensim/models/doc2vec_inner.pyx @@ -9,10 +9,11 @@ import cython import numpy as np +from numpy import zeros, float32 as REAL cimport numpy as np from libc.math cimport exp -from libc.string cimport memset +from libc.string cimport memset, memcpy cdef extern from "voidptr.h": void* PyCObject_AsVoidPtr(object obj) @@ -22,7 +23,7 @@ from scipy.linalg.blas import fblas REAL = np.float32 ctypedef np.float32_t REAL_t -DEF MAX_SENTENCE_LEN = 10000 +DEF MAX_DOCUMENT_LEN = 10000 ctypedef void (*scopy_ptr) (const int *N, const float *X, const int *incX, float *Y, const int *incY) nogil ctypedef void (*saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil @@ -31,39 +32,12 @@ ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, con ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil -ctypedef void (*fast_sentence_dbow_hs_ptr) ( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, int tw, int tl) nogil - -ctypedef unsigned long long (*fast_sentence_dbow_neg_ptr) ( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random, int tw, int tl) nogil - -ctypedef void (*fast_sentence_dm_hs_ptr) ( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - np.uint32_t indexes[MAX_SENTENCE_LEN], np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], const REAL_t alpha, - REAL_t *work, int i, int j, int k, int cbow_mean, int lbl_length, int tw, int tl) nogil - -ctypedef unsigned long long (*fast_sentence_dm_neg_ptr) ( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - np.uint32_t indexes[MAX_SENTENCE_LEN], np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random, int lbl_length, int tw, int tl) nogil - cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x -cdef fast_sentence_dbow_hs_ptr fast_sentence_dbow_hs -cdef fast_sentence_dbow_neg_ptr fast_sentence_dbow_neg -cdef fast_sentence_dm_hs_ptr fast_sentence_dm_hs -cdef fast_sentence_dm_neg_ptr fast_sentence_dm_neg DEF EXP_TABLE_SIZE = 1000 DEF MAX_EXP = 6 @@ -73,92 +47,71 @@ cdef REAL_t[EXP_TABLE_SIZE] EXP_TABLE cdef int ONE = 1 cdef REAL_t ONEF = 1.0 -cdef void fast_sentence0_dbow_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, int tw, int tl) nogil: +# function implementations swapped based on BLAS detected +ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil +ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil - cdef long long a, b - cdef long long row1 = word2_index * size, row2 - cdef REAL_t f, g +cdef our_dot_ptr our_dot +cdef our_saxpy_ptr our_saxpy - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelen): - row2 = word_point[b] * size - f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - if tl: - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - - -cdef void fast_sentence1_dbow_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, int tw, int tl) nogil: +# for when fblas.sdot returns a double +cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + return dsdot(N, X, incX, Y, incY) - cdef long long a, b - cdef long long row1 = word2_index * size, row2 - cdef REAL_t f, g +# for when fblas.sdot returns a float +cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + return sdot(N, X, incX, Y, incY) - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelen): - row2 = word_point[b] * size - f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - if tl: - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) +# for when no blas available +cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + # not a true full dot()-implementation: just enough for our cases + cdef int i + cdef REAL_t a + a = 0.0 + for i from 0 <= i < N[0] by 1: + a += X[i] * Y[i] + return a + +# for when no blas available +cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: + cdef int i + for i from 0 <= i < N[0] by 1: + Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] -cdef void fast_sentence2_dbow_hs( +cdef void fast_document_dbow_hs( const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, int tw, int tl) nogil: + REAL_t *context_vectors, REAL_t *syn1, const int size, + const np.uint32_t context_index, const REAL_t alpha, REAL_t *work, int learn_context, int learn_hidden, + REAL_t *context_locks) nogil: cdef long long a, b - cdef long long row1 = word2_index * size, row2 + cdef long long row1 = context_index * size, row2 cdef REAL_t f, g - for a in range(size): - work[a] = 0.0 + memset(work, 0, size * cython.sizeof(REAL_t)) for b in range(codelen): row2 = word_point[b] * size - f = 0.0 - for a in range(size): - f += syn0[row1 + a] * syn1[row2 + a] + f = our_dot(&size, &context_vectors[row1], &ONE, &syn1[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (1 - word_code[b] - f) * alpha - for a in range(size): - work[a] += g * syn1[row2 + a] - if tw: - for a in range(size): - syn1[row2 + a] += g * syn0[row1 + a] - if tl: - for a in range(size): - syn0[row1 + a] += work[a] + our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1[row2], &ONE) + if learn_context: + our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) -cdef unsigned long long fast_sentence0_dbow_neg( +cdef unsigned long long fast_document_dbow_neg( const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random, int tw, int tl) nogil: + REAL_t *context_vectors, REAL_t *syn1neg, const int size, const np.uint32_t word_index, + const np.uint32_t context_index, const REAL_t alpha, REAL_t *work, + unsigned long long next_random, int learn_context, int learn_hidden, REAL_t *context_locks) nogil: cdef long long a - cdef long long row1 = word2_index * size, row2 + cdef long long row1 = context_index * size, row2 cdef unsigned long long modulo = 281474976710655ULL cdef REAL_t f, g, label cdef np.uint32_t target_index @@ -176,529 +129,308 @@ cdef unsigned long long fast_sentence0_dbow_neg( if target_index == word_index: continue label = 0.0 - row2 = target_index * size - f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + f = our_dot(&size, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - if tl: - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&size, &g, &context_vectors[row1], &ONE, &syn1neg[row2], &ONE) + if learn_context: + our_saxpy(&size, &context_locks[context_index], work, &ONE, &context_vectors[row1], &ONE) return next_random -cdef unsigned long long fast_sentence1_dbow_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random, int tw, int tl) nogil: - - cdef long long a - cdef long long row1 = word2_index * size, row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, label - cdef np.uint32_t target_index - cdef int d - - memset(work, 0, size * cython.sizeof(REAL_t)) - for d in range(negative+1): +cdef void fast_document_dm_hs( + const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, + const int size, int learn_hidden) nogil: - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 + cdef long long b + cdef long long row2 + cdef REAL_t f, g - row2 = target_index * size - f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + # l1 already composed by caller, passed in as neu1 + # work (also passed in) will accumulate l1 error + for b in range(word_code_len): + row2 = word_point[b] * size + f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - if tl: - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + g = (1 - word_code[b] - f) * alpha + our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - return next_random -cdef unsigned long long fast_sentence2_dbow_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random, int tw, int tl) nogil: +cdef unsigned long long fast_document_dm_neg( + const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, + const int size, int learn_hidden) nogil: - cdef long long a - cdef long long row1 = word2_index * size, row2 + cdef long long row2 cdef unsigned long long modulo = 281474976710655ULL cdef REAL_t f, g, label cdef np.uint32_t target_index cdef int d - for a in range(size): - work[a] = 0.0 - + # l1 already composed by caller, passed in as neu1 + # work (also passsed in) will accumulate l1 error for outside application for d in range(negative+1): - if d == 0: - target_index = word_index + target_index = predict_word_index label = ONEF else: target_index = table[(next_random >> 16) % table_len] next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: + if target_index == predict_word_index: continue label = 0.0 row2 = target_index * size - f = 0.0 - for a in range(size): - f += syn0[row1 + a] * syn1neg[row2 + a] + f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (label - f) * alpha - for a in range(size): - work[a] += g * syn1neg[row2 + a] - if tw: - for a in range(size): - syn1neg[row2 + a] += g * syn0[row1 + a] - if tl: - for a in range(size): - syn0[row1 + a] += work[a] + our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) return next_random -cdef void fast_sentence0_dm_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], - const REAL_t alpha, REAL_t *work, int i, int j, int k, int cbow_mean, int lbl_length, int tw, int tl) nogil: - - cdef long long a, b - cdef long long row2 - cdef REAL_t f, g, count, inv_count - cdef int m - - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - - if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelens[i]): - row2 = word_point[b] * size - f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - if tw: - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - -cdef void fast_sentence1_dm_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], - const REAL_t alpha, REAL_t *work, int i, int j, int k, int cbow_mean, int lbl_length, int tw, int tl) nogil: +cdef void fast_document_dmc_hs( + const np.uint32_t *word_point, const np.uint8_t *word_code, int word_code_len, + REAL_t *neu1, REAL_t *syn1, const REAL_t alpha, REAL_t *work, + const int layer1_size, const int vector_size, int learn_hidden) nogil: cdef long long a, b cdef long long row2 - cdef REAL_t f, g, count, inv_count + cdef REAL_t f, g cdef int m - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - - if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count , neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelens[i]): - row2 = word_point[b] * size - f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) + # l1 already composed by caller, passed in as neu1 + # work accumulates net l1 error; eventually applied by caller + for b in range(word_code_len): + row2 = word_point[b] * layer1_size + f = our_dot(&layer1_size, neu1, &ONE, &syn1[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - if tw: - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) - -cdef void fast_sentence2_dm_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], - const REAL_t alpha, REAL_t *work, int i, int j, int k, int cbow_mean, int lbl_length, int tw, int tl) nogil: - - cdef long long a, b - cdef long long row2 - cdef REAL_t f, g, count - cdef int m + our_saxpy(&layer1_size, &g, &syn1[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1[row2], &ONE) - for a in range(size): - neu1[a] = 0.0 - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - for a in range(size): - neu1[a] += syn0[indexes[m] * size + a] - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - for a in range(size): - neu1[a] += syn0[lbl_indexes[m] * size + a] - if cbow_mean and count > (0.5): - for a in range(size): - neu1[a] /= count - - for a in range(size): - work[a] = 0.0 - for b in range(codelens[i]): - row2 = word_point[b] * size - f = 0.0 - for a in range(size): - f += neu1[a] * syn1[row2 + a] - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - for a in range(size): - work[a] += g * syn1[row2 + a] - if tw: - for a in range(size): - syn1[row2 + a] += g * neu1[a] - if tw: - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - for a in range(size): - syn0[indexes[m] * size + a] += work[a] - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - for a in range(size): - syn0[lbl_indexes[m] * size + a] += work[a] - -cdef unsigned long long fast_sentence0_dm_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - np.uint32_t indexes[MAX_SENTENCE_LEN], np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random, int lbl_length, int tw, int tl) nogil: +cdef unsigned long long fast_document_dmc_neg( + const int negative, np.uint32_t *table, unsigned long long table_len, unsigned long long next_random, + REAL_t *neu1, REAL_t *syn1neg, const int predict_word_index, const REAL_t alpha, REAL_t *work, + const int layer1_size, const int vector_size, int learn_hidden) nogil: cdef long long a cdef long long row2 cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label - cdef np.uint32_t target_index, word_index + cdef REAL_t f, g, label + cdef np.uint32_t target_index cdef int d, m - word_index = indexes[i] - - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - + # l1 already composed by caller, passed in as neu1 + # work accumulates net l1 error; eventually applied by caller for d in range(negative+1): if d == 0: - target_index = word_index + target_index = predict_word_index label = ONEF else: target_index = table[(next_random >> 16) % table_len] next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: + if target_index == predict_word_index: continue label = 0.0 - row2 = target_index * size - f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + row2 = target_index * layer1_size + f = our_dot(&layer1_size, neu1, &ONE, &syn1neg[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - if tw: - for m in range(j,k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) + our_saxpy(&layer1_size, &g, &syn1neg[row2], &ONE, work, &ONE) + if learn_hidden: + our_saxpy(&layer1_size, &g, neu1, &ONE, &syn1neg[row2], &ONE) return next_random -cdef unsigned long long fast_sentence1_dm_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - np.uint32_t indexes[MAX_SENTENCE_LEN], np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random, int lbl_length, int tw, int tl) nogil: - cdef long long a - cdef long long row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label - cdef np.uint32_t target_index, word_index - cdef int d, m - - word_index = indexes[i] - - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[lbl_indexes[m] * size], &ONE, neu1, &ONE) - if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - - for d in range(negative+1): - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 +def train_document_dbow(model, word_vocabs, doctag_indexes, alpha, work=None, + train_words=False, learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): + cdef int hs = model.hs + cdef int negative = model.negative + cdef int _train_words = train_words + cdef int _learn_words = learn_words + cdef int _learn_hidden = learn_hidden + cdef int _learn_doctags = learn_doctags + + cdef REAL_t *_word_vectors + cdef REAL_t *_doctag_vectors + cdef REAL_t *_word_locks + cdef REAL_t *_doctag_locks + cdef REAL_t *_work + cdef REAL_t _alpha = alpha + cdef int size = model.layer1_size - row2 = target_index * size - f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - if tw: - saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - if tw: - for m in range(j,k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[lbl_indexes[m]*size], &ONE) + cdef int codelens[MAX_DOCUMENT_LEN] + cdef np.uint32_t indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t _doctag_indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t reduced_windows[MAX_DOCUMENT_LEN] + cdef int document_len + cdef int doctag_len + cdef int window = model.window - return next_random + cdef int i, j + cdef long result = 0 -cdef unsigned long long fast_sentence2_dm_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - int lbl_codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - np.uint32_t indexes[MAX_SENTENCE_LEN], np.uint32_t lbl_indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random, int lbl_length, int tw, int tl) nogil: + # For hierarchical softmax + cdef REAL_t *syn1 + cdef np.uint32_t *points[MAX_DOCUMENT_LEN] + cdef np.uint8_t *codes[MAX_DOCUMENT_LEN] - cdef long long a - cdef long long row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label - cdef np.uint32_t target_index, word_index - cdef int d, m + # For negative sampling + cdef REAL_t *syn1neg + cdef np.uint32_t *table + cdef unsigned long long table_len + cdef unsigned long long next_random - word_index = indexes[i] + # default vectors, locks from syn0/doctag_syn0 + if word_vectors is None: + word_vectors = model.syn0 + _word_vectors = (np.PyArray_DATA(word_vectors)) + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + if word_locks is None: + word_locks = model.syn0_lockf + _word_locks = (np.PyArray_DATA(word_locks)) + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + _doctag_locks = (np.PyArray_DATA(doctag_locks)) - for a in range(size): - neu1[a] = 0.0 - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - for a in range(size): - neu1[a] += syn0[indexes[m] * size + a] - for m in range(lbl_length): - if lbl_codelens[m] == 0: - continue - else: - count += ONEF - for a in range(size): - neu1[a] += syn0[lbl_indexes[m] * size + a] - if cbow_mean and count > (0.5): - for a in range(size): - neu1[a] /= count + if hs: + syn1 = (np.PyArray_DATA(model.syn1)) - for a in range(size): - work[a] = 0.0 + if negative: + syn1neg = (np.PyArray_DATA(model.syn1neg)) + table = (np.PyArray_DATA(model.table)) + table_len = len(model.table) + next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) - for d in range(negative+1): - if d == 0: - target_index = word_index - label = ONEF + # convert Python structures to primitive types, so we can release the GIL + if work is None: + work = zeros(model.layer1_size, dtype=REAL) + _work = np.PyArray_DATA(work) + document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + + for i in range(document_len): + predict_word = word_vocabs[i] + if predict_word is None: + # shrink document to leave out word + document_len = document_len - 1 + continue # leaving j unchanged else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 - - row2 = target_index * size - f = 0.0 - for a in range(size): - f += neu1[a] * syn1neg[row2 + a] - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - for a in range(size): - work[a] += g * syn1neg[row2 + a] - if tw: - for a in range(size): - syn1neg[row2 + a] += g * neu1[a] - - if tw: - for m in range(j, k): - if m == i or codelens[m] == 0: - continue + indexes[i] = predict_word.index + if hs: + codelens[i] = len(predict_word.code) + codes[i] = np.PyArray_DATA(predict_word.code) + points[i] = np.PyArray_DATA(predict_word.point) else: - for a in range(size): - syn0[indexes[m] * size + a] += work[a] - if tl: - for m in range(lbl_length): - if lbl_codelens[m] == 0: + codelens[i] = 1 + result += 1 + if _train_words: + # single randint() call avoids a big thread-synchronization slowdown + for i, item in enumerate(np.random.randint(0, window, document_len)): + reduced_windows[i] = item + for i in range(doctag_len): + _doctag_indexes[i] = doctag_indexes[i] + result += 1 + + # release GIL & train on the document + with nogil: + for i in range(document_len): + if codelens[i] == 0: continue - else: - for a in range(size): - syn0[lbl_indexes[m] * size + a] += work[a] + if _train_words: # simultaneous skip-gram wordvec-training + j = i - window + reduced_windows[i] + if j < 0: + j = 0 + k = i + window + 1 - reduced_windows[i] + if k > document_len: + k = document_len + for j in range(j, k): + if j == i or codelens[j] == 0: + continue + if hs: + # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + fast_document_dbow_hs(points[i], codes[i], codelens[i], _word_vectors, syn1, size, indexes[j], + _alpha, _work, _learn_words, _learn_hidden, _word_locks) + if negative: + # we reuse the DBOW function, as it is equivalent to skip-gram for this purpose + next_random = fast_document_dbow_neg(negative, table, table_len, _word_vectors, syn1neg, size, + indexes[i], indexes[j], _alpha, _work, next_random, + _learn_words, _learn_hidden, _word_locks) + + # docvec-training + for j in range(doctag_len): + if hs: + fast_document_dbow_hs(points[i], codes[i], codelens[i], _doctag_vectors, syn1, size, _doctag_indexes[j], + _alpha, _work, _learn_doctags, _learn_hidden, _doctag_locks) + if negative: + next_random = fast_document_dbow_neg(negative, table, table_len, _doctag_vectors, syn1neg, size, + indexes[i], _doctag_indexes[j], _alpha, _work, next_random, + _learn_doctags, _learn_hidden, _doctag_locks) - return next_random + return result -def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_lbls): + +def train_document_dm(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): cdef int hs = model.hs cdef int negative = model.negative - cdef int tw = train_words - cdef int tl = train_lbls - - cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) - cdef REAL_t *work + cdef int _learn_doctags = learn_doctags + cdef int _learn_words = learn_words + cdef int _learn_hidden = learn_hidden + cdef int cbow_mean = model.cbow_mean + cdef REAL_t count, inv_count = 1.0 + + cdef REAL_t *_word_vectors + cdef REAL_t *_doctag_vectors + cdef REAL_t *_word_locks + cdef REAL_t *_doctag_locks + cdef REAL_t *_work + cdef REAL_t *_neu1 cdef REAL_t _alpha = alpha cdef int size = model.layer1_size - cdef int codelens[MAX_SENTENCE_LEN] - cdef int lbl_codelens[MAX_SENTENCE_LEN] - cdef np.uint32_t indexes[MAX_SENTENCE_LEN] - cdef np.uint32_t lbl_indexes[MAX_SENTENCE_LEN] - cdef np.uint32_t reduced_windows[MAX_SENTENCE_LEN] - cdef int sentence_len - cdef int lbl_length + cdef int codelens[MAX_DOCUMENT_LEN] + cdef np.uint32_t indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t _doctag_indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t reduced_windows[MAX_DOCUMENT_LEN] + cdef int document_len + cdef int doctag_len cdef int window = model.window - cdef int i, j + cdef int i, j, k, m cdef long result = 0 # For hierarchical softmax cdef REAL_t *syn1 - cdef np.uint32_t *points[MAX_SENTENCE_LEN] - cdef np.uint8_t *codes[MAX_SENTENCE_LEN] + cdef np.uint32_t *points[MAX_DOCUMENT_LEN] + cdef np.uint8_t *codes[MAX_DOCUMENT_LEN] # For negative sampling cdef REAL_t *syn1neg @@ -706,6 +438,20 @@ def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_ cdef unsigned long long table_len cdef unsigned long long next_random + # default vectors, locks from syn0/doctag_syn0 + if word_vectors is None: + word_vectors = model.syn0 + _word_vectors = (np.PyArray_DATA(word_vectors)) + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + if word_locks is None: + word_locks = model.syn0_lockf + _word_locks = (np.PyArray_DATA(word_locks)) + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + _doctag_locks = (np.PyArray_DATA(doctag_locks)) + if hs: syn1 = (np.PyArray_DATA(model.syn1)) @@ -713,88 +459,131 @@ def train_sentence_dbow(model, sentence, lbls, alpha, _work, train_words, train_ syn1neg = (np.PyArray_DATA(model.syn1neg)) table = (np.PyArray_DATA(model.table)) table_len = len(model.table) - next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) + next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # convert Python structures to primitive types, so we can release the GIL - work = np.PyArray_DATA(_work) - sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) - lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) - - for i in range(sentence_len): - word = sentence[i] + if work is None: + work = zeros(model.layer1_size, dtype=REAL) + _work = np.PyArray_DATA(work) + if neu1 is None: + neu1 = zeros(model.layer1_size, dtype=REAL) + _neu1 = np.PyArray_DATA(neu1) + + document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + j = 0 + for i in range(document_len): + word = word_vocabs[i] if word is None: - codelens[i] = 0 + # shrink document to leave out word + document_len = document_len - 1 + continue # leaving j unchanged else: - indexes[i] = word.index + indexes[j] = word.index if hs: - codelens[i] = len(word.code) - codes[i] = np.PyArray_DATA(word.code) - points[i] = np.PyArray_DATA(word.point) - else: - codelens[i] = 1 + codelens[j] = len(word.code) + codes[j] = np.PyArray_DATA(word.code) + points[j] = np.PyArray_DATA(word.point) result += 1 + j = j + 1 # single randint() call avoids a big thread-sync slowdown - for i, item in enumerate(np.random.randint(0, window, sentence_len)): + for i, item in enumerate(np.random.randint(0, window, document_len)): reduced_windows[i] = item - for i in range(lbl_length): - word = lbls[i] - if word is None: - lbl_codelens[i] = 0 - else: - lbl_indexes[i] = word.index - if hs: - lbl_codelens[i] = len(word.code) - else: - lbl_codelens[i] = 1 - result += 1 - # release GIL & train on the sentence + doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + for i in range(doctag_len): + _doctag_indexes[i] = doctag_indexes[i] + result += 1 + + # release GIL & train on the document with nogil: - for j in range(lbl_length): - if lbl_codelens[j] == 0: - continue - for i in range(sentence_len): - if codelens[i] == 0: + for i in range(document_len): + j = i - window + reduced_windows[i] + if j < 0: + j = 0 + k = i + window + 1 - reduced_windows[i] + if k > document_len: + k = document_len + + # compose l1 (in _neu1) & clear _work + memset(_neu1, 0, size * cython.sizeof(REAL_t)) + count = 0.0 + for m in range(j, k): + if m == i: continue - if hs: - fast_sentence_dbow_hs(points[i], codes[i], codelens[i], syn0, syn1, size, lbl_indexes[j], _alpha, work, tw, tl) - if negative: - next_random = fast_sentence_dbow_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], lbl_indexes[j], _alpha, work, next_random, tw, tl) + else: + count += ONEF + our_saxpy(&size, &ONEF, &_word_vectors[indexes[m] * size], &ONE, _neu1, &ONE) + for m in range(doctag_len): + count += ONEF + our_saxpy(&size, &ONEF, &_doctag_vectors[_doctag_indexes[m] * size], &ONE, _neu1, &ONE) + if count > (0.5): + inv_count = ONEF/count + if cbow_mean: + sscal(&size, &inv_count, _neu1, &ONE) # (does this need BLAS-variants like saxpy?) + memset(_work, 0, size * cython.sizeof(REAL_t)) # work to accumulate l1 error + if hs: + fast_document_dm_hs(points[i], codes[i], codelens[i], + _neu1, syn1, _alpha, _work, + size, _learn_hidden) + if negative: + next_random = fast_document_dm_neg(negative, table, table_len, next_random, + _neu1, syn1neg, indexes[i], _alpha, _work, + size, _learn_hidden) + + if not cbow_mean: + sscal(&size, &inv_count, _work, &ONE) # (does this need BLAS-variants like saxpy?) + # apply accumulated error in work + if _learn_doctags: + for m in range(doctag_len): + our_saxpy(&size, &_doctag_locks[_doctag_indexes[m]], _work, + &ONE, &_doctag_vectors[_doctag_indexes[m] * size], &ONE) + if _learn_words: + for m in range(j, k): + if m == i: + continue + else: + our_saxpy(&size, &_word_locks[indexes[m]], _work, &ONE, + &_word_vectors[indexes[m] * size], &ONE) return result -def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, train_lbls): +def train_document_dm_concat(model, word_vocabs, doctag_indexes, alpha, work=None, neu1=None, + learn_doctags=True, learn_words=True, learn_hidden=True, + word_vectors=None, word_locks=None, doctag_vectors=None, doctag_locks=None): cdef int hs = model.hs cdef int negative = model.negative - cdef int tw = train_words - cdef int tl = train_lbls - cdef int cbow_mean = model.cbow_mean - - cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) - cdef REAL_t *work - cdef REAL_t *neu1 + cdef int _learn_doctags = learn_doctags + cdef int _learn_words = learn_words + cdef int _learn_hidden = learn_hidden + + cdef REAL_t *_word_vectors + cdef REAL_t *_doctag_vectors + cdef REAL_t *_word_locks + cdef REAL_t *_doctag_locks + cdef REAL_t *_work + cdef REAL_t *_neu1 cdef REAL_t _alpha = alpha - cdef int size = model.layer1_size - - cdef int codelens[MAX_SENTENCE_LEN] - cdef int lbl_codelens[MAX_SENTENCE_LEN] - cdef np.uint32_t indexes[MAX_SENTENCE_LEN] - cdef np.uint32_t lbl_indexes[MAX_SENTENCE_LEN] - cdef np.uint32_t reduced_windows[MAX_SENTENCE_LEN] - cdef int sentence_len - cdef int lbl_length + cdef int layer1_size = model.layer1_size + cdef int vector_size = model.vector_size + + cdef int codelens[MAX_DOCUMENT_LEN] + cdef np.uint32_t indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t _doctag_indexes[MAX_DOCUMENT_LEN] + cdef np.uint32_t window_indexes[MAX_DOCUMENT_LEN] + cdef int document_len + cdef int doctag_len cdef int window = model.window + cdef int expected_doctag_len = model.dm_tag_count - cdef int i, j, k + cdef int i, j, k, m, n cdef long result = 0 + cdef int null_word_index = model.vocab['\0'].index # For hierarchical softmax cdef REAL_t *syn1 - cdef np.uint32_t *points[MAX_SENTENCE_LEN] - cdef np.uint8_t *codes[MAX_SENTENCE_LEN] - cdef np.uint32_t *lbl_points[MAX_SENTENCE_LEN] - cdef np.uint8_t *lbl_codes[MAX_SENTENCE_LEN] + cdef np.uint32_t *points[MAX_DOCUMENT_LEN] + cdef np.uint8_t *codes[MAX_DOCUMENT_LEN] # For negative sampling cdef REAL_t *syn1neg @@ -802,6 +591,24 @@ def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, t cdef unsigned long long table_len cdef unsigned long long next_random + doctag_len = min(MAX_DOCUMENT_LEN, len(doctag_indexes)) + if doctag_len != expected_doctag_len: + return 0 # skip doc without expected nmber of tags + + # default vectors, locks from syn0/doctag_syn0 + if word_vectors is None: + word_vectors = model.syn0 + _word_vectors = (np.PyArray_DATA(word_vectors)) + if doctag_vectors is None: + doctag_vectors = model.docvecs.doctag_syn0 + _doctag_vectors = (np.PyArray_DATA(doctag_vectors)) + if word_locks is None: + word_locks = model.syn0_lockf + _word_locks = (np.PyArray_DATA(word_locks)) + if doctag_locks is None: + doctag_locks = model.docvecs.doctag_syn0_lockf + _doctag_locks = (np.PyArray_DATA(doctag_locks)) + if hs: syn1 = (np.PyArray_DATA(model.syn1)) @@ -812,58 +619,79 @@ def train_sentence_dm(model, sentence, lbls, alpha, _work, _neu1, train_words, t next_random = (2**24)*np.random.randint(0,2**24) + np.random.randint(0,2**24) # convert Python structures to primitive types, so we can release the GIL - work = np.PyArray_DATA(_work) - neu1 = np.PyArray_DATA(_neu1) - sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) - - for i in range(sentence_len): - word = sentence[i] + if work is None: + work = zeros(model.layer1_size, dtype=REAL) + _work = np.PyArray_DATA(work) + if neu1 is None: + neu1 = zeros(model.layer1_size, dtype=REAL) + _neu1 = np.PyArray_DATA(neu1) + + document_len = min(MAX_DOCUMENT_LEN, len(word_vocabs)) + j = 0 + for i in range(document_len): + word = word_vocabs[i] if word is None: - codelens[i] = 0 + # shrink document to leave out word + document_len = document_len - 1 + continue # leaving j unchanged else: - indexes[i] = word.index + indexes[j] = word.index if hs: - codelens[i] = len(word.code) - codes[i] = np.PyArray_DATA(word.code) - points[i] = np.PyArray_DATA(word.point) + codelens[j] = len(word.code) + codes[j] = np.PyArray_DATA(word.code) + points[j] = np.PyArray_DATA(word.point) else: - codelens[i] = 1 + codelens[j] = 1 result += 1 - # single randint() call avoids a big thread-sync slowdown - for i, item in enumerate(np.random.randint(0, window, sentence_len)): - reduced_windows[i] = item + j = j + 1 - lbl_length = min(MAX_SENTENCE_LEN, len(lbls)) - for i in range(lbl_length): - word = lbls[i] - if word is None: - lbl_codelens[i] = 0 - else: - lbl_indexes[i] = word.index - if hs: - lbl_codelens[i] = len(word.code) - else: - lbl_codelens[i] = 1 - result += 1 + for i in range(doctag_len): + _doctag_indexes[i] = doctag_indexes[i] + result += 1 - # release GIL & train on the sentence + # release GIL & train on the document with nogil: - for i in range(sentence_len): - if codelens[i] == 0: - continue - j = i - window + reduced_windows[i] - if j < 0: - j = 0 - k = i + window + 1 - reduced_windows[i] - if k > sentence_len: - k = sentence_len + for i in range(document_len): + j = i - window # negative OK: will pad with null word + k = i + window + 1 # past document end OK: will pad with null word + + # compose l1 & clear work + for m in range(doctag_len): + # doc vector(s) + memcpy(&_neu1[m * vector_size], &_doctag_vectors[_doctag_indexes[m] * vector_size], + vector_size * cython.sizeof(REAL_t)) + n = 0 + for m in range(j, k): + # word vectors in window + if m == i: + continue + if m < 0 or m >= document_len: + window_indexes[n] = null_word_index + else: + window_indexes[n] = indexes[m] + n = n + 1 + for m in range(2 * window): + memcpy(&_neu1[(doctag_len + m) * vector_size], &_word_vectors[window_indexes[m] * vector_size], + vector_size * cython.sizeof(REAL_t)) + memset(_work, 0, layer1_size * cython.sizeof(REAL_t)) # work to accumulate l1 error + if hs: - fast_sentence_dm_hs(points[i], codes[i], codelens, lbl_codelens, neu1, syn0, syn1, size, indexes, - lbl_indexes, _alpha, work, i, j, k, cbow_mean, lbl_length, tw, tl) + fast_document_dmc_hs(points[i], codes[i], codelens[i], + _neu1, syn1, _alpha, _work, + layer1_size, vector_size, _learn_hidden) if negative: - next_random = fast_sentence_dm_neg(negative, table, table_len, codelens, lbl_codelens, neu1, syn0, - syn1neg, size, indexes, lbl_indexes, _alpha, work, i, j, k, - cbow_mean, next_random, lbl_length, tw, tl) + next_random = fast_document_dmc_neg(negative, table, table_len, next_random, + _neu1, syn1neg, indexes[i], _alpha, _work, + layer1_size, vector_size, _learn_hidden) + + if _learn_doctags: + for m in range(doctag_len): + our_saxpy(&vector_size, &_doctag_locks[_doctag_indexes[m]], &_work[m * vector_size], + &ONE, &_doctag_vectors[_doctag_indexes[m] * vector_size], &ONE) + if _learn_words: + for m in range(2 * window): + our_saxpy(&vector_size, &_word_locks[window_indexes[m]], &_work[(doctag_len + m) * vector_size], + &ONE, &_word_vectors[window_indexes[m] * vector_size], &ONE) return result @@ -874,10 +702,8 @@ def init(): into table EXP_TABLE. """ - global fast_sentence_dbow_hs - global fast_sentence_dbow_neg - global fast_sentence_dm_hs - global fast_sentence_dm_neg + global our_dot + global our_saxpy cdef int i cdef float *x = [10.0] @@ -896,24 +722,18 @@ def init(): d_res = dsdot(&size, x, &ONE, y, &ONE) p_res = &d_res if (abs(d_res - expected) < 0.0001): - fast_sentence_dbow_hs = fast_sentence0_dbow_hs - fast_sentence_dbow_neg = fast_sentence0_dbow_neg - fast_sentence_dm_hs = fast_sentence0_dm_hs - fast_sentence_dm_neg = fast_sentence0_dm_neg + our_dot = our_dot_double + our_saxpy = saxpy return 0 # double elif (abs(p_res[0] - expected) < 0.0001): - fast_sentence_dbow_hs = fast_sentence1_dbow_hs - fast_sentence_dbow_neg = fast_sentence1_dbow_neg - fast_sentence_dm_hs = fast_sentence1_dm_hs - fast_sentence_dm_neg = fast_sentence1_dm_neg + our_dot = our_dot_float + our_saxpy = saxpy return 1 # float else: # neither => use cython loops, no BLAS # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - fast_sentence_dbow_hs = fast_sentence2_dbow_hs - fast_sentence_dbow_neg = fast_sentence2_dbow_neg - fast_sentence_dm_hs = fast_sentence2_dm_hs - fast_sentence_dm_neg = fast_sentence2_dm_neg + our_dot = our_dot_noblas + our_saxpy = our_saxpy_noblas return 2 FAST_VERSION = init() # initialize the module diff --git a/gensim/models/word2vec.py b/gensim/models/word2vec.py index 3ff49909d9..bf76a28c4e 100755 --- a/gensim/models/word2vec.py +++ b/gensim/models/word2vec.py @@ -81,7 +81,7 @@ from numpy import exp, dot, zeros, outer, random, dtype, float32 as REAL,\ uint32, seterr, array, uint8, vstack, argsort, fromstring, sqrt, newaxis,\ - ndarray, empty, sum as np_sum, prod + ndarray, empty, sum as np_sum, prod, ones, repeat as np_repeat logger = logging.getLogger("gensim.models.word2vec") @@ -109,12 +109,6 @@ def train_sentence_sg(model, sentence, alpha, work=None): will use the optimized version from word2vec_inner instead. """ - labels = [] - if model.negative: - # precompute negative labels - labels = zeros(model.negative + 1) - labels[0] = 1.0 - for pos, word in enumerate(sentence): if word is None: continue # OOV word in the input sentence => skip @@ -125,7 +119,7 @@ def train_sentence_sg(model, sentence, alpha, work=None): for pos2, word2 in enumerate(sentence[start : pos + model.window + 1 - reduced_window], start): # don't train on OOV words and on the `word` itself if word2 and not (pos2 == pos): - train_sg_pair(model, word, word2, alpha, labels) + train_sg_pair(model, word, word2.index, alpha) return len([word for word in sentence if word is not None]) @@ -140,12 +134,6 @@ def train_sentence_cbow(model, sentence, alpha, work=None, neu1=None): will use the optimized version from word2vec_inner instead. """ - labels = [] - if model.negative: - # precompute negative labels - labels = zeros(model.negative + 1) - labels[0] = 1. - for pos, word in enumerate(sentence): if word is None: continue # OOV word in the input sentence => skip @@ -153,53 +141,61 @@ def train_sentence_cbow(model, sentence, alpha, work=None, neu1=None): start = max(0, pos - model.window + reduced_window) window_pos = enumerate(sentence[start : pos + model.window + 1 - reduced_window], start) word2_indices = [word2.index for pos2, word2 in window_pos if (word2 is not None and pos2 != pos)] - l1 = np_sum(model.syn0[word2_indices], axis=0) # 1 x layer1_size + l1 = np_sum(model.syn0[word2_indices], axis=0) # 1 x vector_size if word2_indices and model.cbow_mean: l1 /= len(word2_indices) - train_cbow_pair(model, word, word2_indices, l1, alpha, labels) + train_cbow_pair(model, word, word2_indices, l1, alpha) return len([word for word in sentence if word is not None]) -def train_sg_pair(model, word, word2, alpha, labels, train_w1=True, train_w2=True): - l1 = model.syn0[word2.index] +def train_sg_pair(model, predict_word, context_index, alpha, learn_vectors=True, learn_hidden=True, + context_vectors=None, context_locks=None): + if context_vectors is None: + context_vectors = model.syn0 + if context_locks is None: + context_locks = model.syn0_lockf + + l1 = context_vectors[context_index] + lock_factor = context_locks[context_index] + neu1e = zeros(l1.shape) if model.hs: # work on the entire tree at once, to push as much work into numpy's C routines as possible (performance) - l2a = deepcopy(model.syn1[word.point]) # 2d matrix, codelen x layer1_size + l2a = deepcopy(model.syn1[predict_word.point]) # 2d matrix, codelen x layer1_size fa = 1.0 / (1.0 + exp(-dot(l1, l2a.T))) # propagate hidden -> output - ga = (1 - word.code - fa) * alpha # vector of error gradients multiplied by the learning rate - if train_w1: - model.syn1[word.point] += outer(ga, l1) # learn hidden -> output + ga = (1 - predict_word.code - fa) * alpha # vector of error gradients multiplied by the learning rate + if learn_hidden: + model.syn1[predict_word.point] += outer(ga, l1) # learn hidden -> output neu1e += dot(ga, l2a) # save error if model.negative: # use this word (label = 1) + `negative` other random words not from this sentence (label = 0) - word_indices = [word.index] + word_indices = [predict_word.index] while len(word_indices) < model.negative + 1: w = model.table[random.randint(model.table.shape[0])] - if w != word.index: + if w != predict_word.index: word_indices.append(w) l2b = model.syn1neg[word_indices] # 2d matrix, k+1 x layer1_size fb = 1. / (1. + exp(-dot(l1, l2b.T))) # propagate hidden -> output - gb = (labels - fb) * alpha # vector of error gradients multiplied by the learning rate - if train_w1: + gb = (model.neg_labels - fb) * alpha # vector of error gradients multiplied by the learning rate + if learn_hidden: model.syn1neg[word_indices] += outer(gb, l1) # learn hidden -> output neu1e += dot(gb, l2b) # save error - if train_w2: - model.syn0[word2.index] += neu1e # learn input -> hidden + if learn_vectors: + l1 += neu1e * lock_factor # learn input -> hidden (mutates model.syn0[word2.index], if that is l1) return neu1e -def train_cbow_pair(model, word, word2_indices, l1, alpha, labels, train_w1=True, train_w2=True): +def train_cbow_pair(model, word, input_word_indices, l1, alpha, learn_vectors=True, learn_hidden=True): neu1e = zeros(l1.shape) if model.hs: l2a = model.syn1[word.point] # 2d matrix, codelen x layer1_size fa = 1. / (1. + exp(-dot(l1, l2a.T))) # propagate hidden -> output ga = (1. - word.code - fa) * alpha # vector of error gradients multiplied by the learning rate - if train_w1: + if learn_hidden: model.syn1[word.point] += outer(ga, l1) # learn hidden -> output neu1e += dot(ga, l2a) # save error @@ -212,12 +208,14 @@ def train_cbow_pair(model, word, word2_indices, l1, alpha, labels, train_w1=True word_indices.append(w) l2b = model.syn1neg[word_indices] # 2d matrix, k+1 x layer1_size fb = 1. / (1. + exp(-dot(l1, l2b.T))) # propagate hidden -> output - gb = (labels - fb) * alpha # vector of error gradients multiplied by the learning rate - if train_w1: + gb = (model.neg_labels - fb) * alpha # vector of error gradients multiplied by the learning rate + if learn_hidden: model.syn1neg[word_indices] += outer(gb, l1) # learn hidden -> output neu1e += dot(gb, l2b) # save error - if train_w2: - model.syn0[word2_indices] += neu1e # learn input -> hidden, here for all words in the window separately + if learn_vectors: + # learn input -> hidden, here for all words in the window separately + l = len(input_word_indices) + model.syn0[input_word_indices] += np_repeat(neu1e,l).reshape(l,model.vector_size) * model.syn0_lockf[input_word_indices][:,None] return neu1e @@ -245,7 +243,7 @@ class Word2Vec(utils.SaveLoad): """ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5, sample=0, seed=1, workers=1, min_alpha=0.0001, sg=1, hs=1, negative=0, - cbow_mean=0, hashfxn=hash, iter=1): + cbow_mean=0, hashfxn=hash, iter=1, null_word=0): """ Initialize the model from an iterable of `sentences`. Each sentence is a list of words (unicode strings) that will be used for training. @@ -294,6 +292,7 @@ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5, self.index2word = [] # map from a word's matrix index (int) to word (string) self.sg = int(sg) self.table = None # for negative sampling --> this needs a lot of RAM! consider setting back to None before saving + self.vector_size = int(size) self.layer1_size = int(size) if size % 4 != 0: logger.warning("consider setting layer size to a multiple of 4 for greater performance") @@ -309,6 +308,7 @@ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5, self.cbow_mean = int(cbow_mean) self.hashfxn = hashfxn self.iter = iter + self.null_word = null_word if sentences is not None: if isinstance(sentences, GeneratorType): raise TypeError("You can't pass a generator as the sentences argument. Try an iterator.") @@ -334,7 +334,7 @@ def make_table(self, table_size=100000000, power=0.75): # compute sum of all power (Z in paper) train_words_pow = float(sum([self.vocab[word].count**power for word in self.vocab])) - # go through the whole table and fill it up with the word indexes proportional to a word's count**power + # go through the whole table and fill it up with the word indices proportional to a word's count**power widx = 0 # normalize count^0.75 by Z d1 = self.vocab[self.index2word[widx]].count**power / train_words_pow @@ -405,6 +405,13 @@ def build_vocab(self, sentences): self.vocab[word] = v logger.info("total %i word types after removing those with count<%s" % (len(self.vocab), self.min_count)) + if self.null_word: + # create null pseudo-word for padding when using concatenative L1 (run-of-words) + # this word is only ever input – never predicted – so count, huffman-point doesn't matter + word, v = '\0', Vocab(count=1) + v.index = len(self.vocab) + self.index2word.append(word) + self.vocab[word] = v if self.hs: # add info about each word's Huffman encoding self.create_binary_tree() @@ -414,9 +421,9 @@ def build_vocab(self, sentences): # precalculate downsampling thresholds self.precalc_sampling() self.reset_weights() + sys.stderr.flush() - @staticmethod - def _vocab_from(sentences): + def _vocab_from(self, sentences): sentence_no, vocab = -1, {} total_words = 0 for sentence_no, sentence in enumerate(sentences): @@ -433,8 +440,18 @@ def _vocab_from(sentences): (len(vocab), total_words, sentence_no + 1)) return vocab - def _prepare_sentences(self, sentences): - for sentence in sentences: + def reset_from(self, other_model): + """ + Borrow shareable pre-built structures (like vocab) from the other_model. Useful + if testing multiple models in parallel on the same corpus. + """ + self.vocab = other_model.vocab + self.index2word = other_model.index2word + self.table = other_model.table + self.reset_weights() + + def _prepare_items(self, items): + for sentence in items: # avoid calling random_sample() where prob >= 1, to speed things up a little: sampled = [self.vocab[word] for word in sentence if word in self.vocab and (self.vocab[word].sample_probability >= 1.0 or @@ -456,6 +473,12 @@ def train(self, sentences, total_words=None, word_count=0, chunksize=100): if FAST_VERSION < 0: import warnings warnings.warn("C extension compilation failed, training will be slow. Install a C compiler and reinstall gensim for fast training.") + self.neg_labels = [] + if self.negative > 0: + # precompute negative labels optimization for pure-python training + self.neg_labels = zeros(self.negative + 1) + self.neg_labels[0] = 1. + logger.info("training model with %i workers on %i vocabulary and %i features, " "using 'skipgram'=%s 'hierarchical softmax'=%s 'subsample'=%s and 'negative sampling'=%s" % (self.workers, len(self.vocab), self.layer1_size, self.sg, self.hs, self.sample, self.negative)) @@ -492,6 +515,7 @@ def worker_train(): logger.info("PROGRESS: at %.2f%% words, alpha %.05f, %.0f words/s" % (100.0 * word_count[0] / total_words, alpha, word_count[0] / elapsed if elapsed else 0.0)) next_report[0] = elapsed + 1.0 # don't flood the log, wait at least a second between progress reports + sys.stderr.flush() workers = [threading.Thread(target=worker_train) for _ in xrange(self.workers)] for thread in workers: @@ -499,7 +523,7 @@ def worker_train(): thread.start() # convert input strings to Vocab objects (eliding OOV/downsampled words), and start filling the jobs queue - for job_no, job in enumerate(utils.grouper(self._prepare_sentences(sentences), chunksize)): + for job_no, job in enumerate(utils.grouper(self._prepare_items(sentences), chunksize)): logger.debug("putting job #%i in the queue, qsize=%i" % (job_no, jobs.qsize())) jobs.put(job) logger.info("reached the end of input; waiting to finish %i outstanding jobs" % jobs.qsize()) @@ -512,25 +536,33 @@ def worker_train(): elapsed = time.time() - start logger.info("training on %i words took %.1fs, %.0f words/s" % (word_count[0], elapsed, word_count[0] / elapsed if elapsed else 0.0)) - self.syn0norm = None + self.clear_sims() return word_count[0] + def clear_sims(self): + self.syn0norm = None + def reset_weights(self): """Reset all projection weights to an initial (untrained) state, but keep the existing vocabulary.""" logger.info("resetting layer weights") - self.syn0 = empty((len(self.vocab), self.layer1_size), dtype=REAL) + self.syn0 = empty((len(self.vocab), self.vector_size), dtype=REAL) # randomize weights vector by vector, rather than materializing a huge random matrix in RAM at once for i in xrange(len(self.vocab)): # construct deterministic seed from word AND seed argument - # Note: Python's built in hash function can vary across versions of Python - random.seed(uint32(self.hashfxn(self.index2word[i] + str(self.seed)))) - self.syn0[i] = (random.rand(self.layer1_size) - 0.5) / self.layer1_size + self.syn0[i] = self.seeded_vector(self.index2word[i] + str(self.seed)) if self.hs: self.syn1 = zeros((len(self.vocab), self.layer1_size), dtype=REAL) if self.negative: self.syn1neg = zeros((len(self.vocab), self.layer1_size), dtype=REAL) self.syn0norm = None + self.syn0_lockf = ones(len(self.vocab), dtype=REAL) # zeros suppress learning + + def seeded_vector(self, seed_string): + """Create one 'random' vector (but deterministic by seed_string)""" + # Note: Python's built in hash function can vary across versions of Python + random.seed(uint32(self.hashfxn(seed_string))) + return (random.rand(self.vector_size) - 0.5) / self.vector_size def save_word2vec_format(self, fname, fvocab=None, binary=False): """ @@ -543,8 +575,8 @@ def save_word2vec_format(self, fname, fvocab=None, binary=False): with utils.smart_open(fvocab, 'wb') as vout: for word, vocab in sorted(iteritems(self.vocab), key=lambda item: -item[1].count): vout.write(utils.to_utf8("%s %s\n" % (word, vocab.count))) - logger.info("storing %sx%s projection weights into %s" % (len(self.vocab), self.layer1_size, fname)) - assert (len(self.vocab), self.layer1_size) == self.syn0.shape + logger.info("storing %sx%s projection weights into %s" % (len(self.vocab), self.vector_size, fname)) + assert (len(self.vocab), self.vector_size) == self.syn0.shape with utils.smart_open(fname, 'wb') as fout: fout.write(utils.to_utf8("%s %s\n" % self.syn0.shape)) # store in sorted order: most frequent words at the top @@ -582,11 +614,11 @@ def load_word2vec_format(cls, fname, fvocab=None, binary=False, norm_only=True): logger.info("loading projection weights from %s" % (fname)) with utils.smart_open(fname) as fin: header = utils.to_unicode(fin.readline()) - vocab_size, layer1_size = map(int, header.split()) # throws for invalid file format - result = Word2Vec(size=layer1_size) - result.syn0 = zeros((vocab_size, layer1_size), dtype=REAL) + vocab_size, vector_size = map(int, header.split()) # throws for invalid file format + result = Word2Vec(size=vector_size) + result.syn0 = zeros((vocab_size, vector_size), dtype=REAL) if binary: - binary_len = dtype(REAL).itemsize * layer1_size + binary_len = dtype(REAL).itemsize * vector_size for line_no in xrange(vocab_size): # mixed text and binary: read text first, then binary word = [] @@ -609,7 +641,7 @@ def load_word2vec_format(cls, fname, fvocab=None, binary=False, norm_only=True): else: for line_no, line in enumerate(fin): parts = utils.to_unicode(line).split() - if len(parts) != layer1_size + 1: + if len(parts) != vector_size + 1: raise ValueError("invalid vector on line %s (is this really the text format?)" % (line_no)) word, weights = parts[0], list(map(REAL, parts[1:])) if counts is None: @@ -625,6 +657,53 @@ def load_word2vec_format(cls, fname, fvocab=None, binary=False, norm_only=True): result.init_sims(norm_only) return result + def intersect_word2vec_format(self, fname, binary=False): + """ + Merge the input-hidden weight matrix from the original C word2vec-tool format + given, where it intersects with the current vocabulary. (No words are added to the + existing vocabulary, but intersecting words adopt the file's weights, and + non-intersecting words are left alone.) + + `binary` is a boolean indicating whether the data is in binary word2vec format. + """ + counts = None + overlap_count = 0 + logger.info("loading projection weights from %s" % (fname)) + with utils.smart_open(fname) as fin: + header = utils.to_unicode(fin.readline()) + vocab_size, vector_size = map(int, header.split()) # throws for invalid file format + if not vector_size == self.vector_size: + logger.error("incompatible vector sizes") + # TOCONSIDER: maybe truncation/smaller vectors still useful enough to merge? + return # TODO raise ValueError()? + if binary: + binary_len = dtype(REAL).itemsize * vector_size + for line_no in xrange(vocab_size): + # mixed text and binary: read text first, then binary + word = [] + while True: + ch = fin.read(1) + if ch == b' ': + break + if ch != b'\n': # ignore newlines in front of words (some binary files have newline, some don't) + word.append(ch) + word = utils.to_unicode(b''.join(word)) + weights = fromstring(fin.read(binary_len), dtype=REAL) + if word in self.vocab: + overlap_count += 1 + self.syn0[self.vocab[word].index] = weights + self.syn0_lockf[self.vocab[word].index] = 0.0 # lock it + else: + for line_no, line in enumerate(fin): + parts = utils.to_unicode(line).split() + if len(parts) != vector_size + 1: + raise ValueError("invalid vector on line %s (is this really the text format?)" % (line_no)) + word, weights = parts[0], list(map(REAL, parts[1:])) + if word in self.vocab: + overlap_count += 1 + self.syn0[self.vocab[word].index] = weights + logger.info("merged %d vectors into %s matrix from %s" % (overlap_count, self.syn0.shape, fname)) + def most_similar(self, positive=[], negative=[], topn=10): """ @@ -910,15 +989,24 @@ def accuracy(self, questions, restrict_vocab=30000, most_similar=most_similar): def __str__(self): - return "Word2Vec(vocab=%s, size=%s, alpha=%s)" % (len(self.index2word), self.layer1_size, self.alpha) + return "Word2Vec(vocab=%s, size=%s, alpha=%s)" % (len(self.index2word), self.vector_size, self.alpha) def save(self, *args, **kwargs): - kwargs['ignore'] = kwargs.get('ignore', ['syn0norm']) # don't bother storing the cached normalized vectors + # don't bother storing the cached normalized vectors, recalculable table + kwargs['ignore'] = kwargs.get('ignore', ['syn0norm','table']) super(Word2Vec, self).save(*args, **kwargs) save.__doc__ = utils.SaveLoad.save.__doc__ + + @classmethod + def load(cls, *args, **kwargs): + model = super(Word2Vec, cls).load(*args, **kwargs) + if model.negative and not model.table: + model.make_table() # rebuild table if missing + return model + class BrownCorpus(object): """Iterate over sentences from the Brown corpus (part of NLTK data).""" def __init__(self, dirname): diff --git a/gensim/models/word2vec_inner.c b/gensim/models/word2vec_inner.c index f369134f5c..682930f1d9 100644 --- a/gensim/models/word2vec_inner.c +++ b/gensim/models/word2vec_inner.c @@ -765,45 +765,27 @@ typedef double (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_snrm2_ptr)(int * ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil # <<<<<<<<<<<<<< * - * ctypedef void (*fast_sentence_sg_hs_ptr) ( + * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x */ typedef void (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_sscal_ptr)(int const *, float const *, float const *, int const *); -/* "trunk/gensim/models/word2vec_inner.pyx":34 - * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil - * - * ctypedef void (*fast_sentence_sg_hs_ptr) ( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, - */ -typedef void (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs_ptr)(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); - -/* "trunk/gensim/models/word2vec_inner.pyx":39 - * const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work) nogil +/* "trunk/gensim/models/word2vec_inner.pyx":50 * - * ctypedef unsigned long long (*fast_sentence_sg_neg_ptr) ( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ -typedef unsigned PY_LONG_LONG (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg_ptr)(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, unsigned PY_LONG_LONG); - -/* "trunk/gensim/models/word2vec_inner.pyx":45 - * unsigned long long next_random) nogil + * # function implementations swapped based on BLAS detected + * ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil # <<<<<<<<<<<<<< + * ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil * - * ctypedef void (*fast_sentence_cbow_hs_ptr) ( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, */ -typedef void (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs_ptr)(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int); +typedef __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_our_dot_ptr)(int const *, float const *, int const *, float const *, int const *); /* "trunk/gensim/models/word2vec_inner.pyx":51 - * int i, int j, int k, int cbow_mean) nogil + * # function implementations swapped based on BLAS detected + * ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil + * ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil # <<<<<<<<<<<<<< * - * ctypedef unsigned long long (*fast_sentence_cbow_neg_ptr) ( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * cdef our_dot_ptr our_dot */ -typedef unsigned PY_LONG_LONG (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg_ptr)(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG); +typedef void (*__pyx_t_5trunk_6gensim_6models_14word2vec_inner_our_saxpy_ptr)(int const *, float const *, float const *, int const *, float *, int const *); /* --- Runtime support code (head) --- */ #ifndef CYTHON_REFNANNY @@ -1160,25 +1142,19 @@ static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_sdot_ptr __pyx_v_5trunk_6 static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_dsdot_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot; static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_snrm2_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_snrm2; static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_sscal_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal; -static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs; -static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg; -static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs; -static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg; static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[1000]; static int __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE; static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, unsigned PY_LONG_LONG); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, unsigned PY_LONG_LONG); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, unsigned PY_LONG_LONG); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int); /*proto*/ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG); /*proto*/ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_our_dot_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot; +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_our_saxpy_ptr __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy; +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_double(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_float(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_noblas(int const *, float const *, int const *, float const *, int const *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_saxpy_noblas(int const *, float const *, float const *, int const *, float *, int const *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const , __pyx_t_5numpy_uint32_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs(__pyx_t_5numpy_uint32_t const *, __pyx_t_5numpy_uint8_t const *, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg(int const , __pyx_t_5numpy_uint32_t *, unsigned PY_LONG_LONG, int *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int const , __pyx_t_5numpy_uint32_t const *, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const , __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *, int, int, int, int, unsigned PY_LONG_LONG, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *); /*proto*/ #define __Pyx_MODULE_NAME "trunk.gensim.models.word2vec_inner" int __pyx_module_is_main_trunk__gensim__models__word2vec_inner = 0; @@ -1264,6 +1240,8 @@ static char __pyx_k_cbow_mean[] = "cbow_mean"; static char __pyx_k_enumerate[] = "enumerate"; static char __pyx_k_table_len[] = "table_len"; static char __pyx_k_ValueError[] = "ValueError"; +static char __pyx_k_syn0_lockf[] = "syn0_lockf"; +static char __pyx_k_word_locks[] = "word_locks"; static char __pyx_k_layer1_size[] = "layer1_size"; static char __pyx_k_next_random[] = "next_random"; static char __pyx_k_FAST_VERSION[] = "FAST_VERSION"; @@ -1339,6 +1317,7 @@ static PyObject *__pyx_n_s_size; static PyObject *__pyx_n_s_snrm2; static PyObject *__pyx_n_s_sscal; static PyObject *__pyx_n_s_syn0; +static PyObject *__pyx_n_s_syn0_lockf; static PyObject *__pyx_n_s_syn1; static PyObject *__pyx_n_s_syn1neg; static PyObject *__pyx_n_s_table; @@ -1350,6 +1329,7 @@ static PyObject *__pyx_n_s_trunk_gensim_models_word2vec_inn; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_window; static PyObject *__pyx_n_s_word; +static PyObject *__pyx_n_s_word_locks; static PyObject *__pyx_n_s_work; static PyObject *__pyx_n_s_work_2; static PyObject *__pyx_n_s_x; @@ -1375,167 +1355,192 @@ static PyObject *__pyx_codeobj__12; static PyObject *__pyx_codeobj__14; static PyObject *__pyx_codeobj__16; -/* "trunk/gensim/models/word2vec_inner.pyx":76 - * cdef REAL_t ONEF = 1.0 +/* "trunk/gensim/models/word2vec_inner.pyx":57 + * + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return dsdot(N, X, incX, Y, incY) * - * cdef void fast_sentence0_sg_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, */ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row1; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_double(int const *__pyx_v_N, float const *__pyx_v_X, int const *__pyx_v_incX, float const *__pyx_v_Y, int const *__pyx_v_incY) { + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_r; - /* "trunk/gensim/models/word2vec_inner.pyx":82 + /* "trunk/gensim/models/word2vec_inner.pyx":58 + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + * return dsdot(N, X, incX, Y, incY) # <<<<<<<<<<<<<< * - * cdef long long a, b - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef REAL_t f, g + * # for when fblas.sdot returns a float + */ + __pyx_r = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot(__pyx_v_N, __pyx_v_X, __pyx_v_incX, __pyx_v_Y, __pyx_v_incY)); + goto __pyx_L0; + + /* "trunk/gensim/models/word2vec_inner.pyx":57 + * + * # for when fblas.sdot returns a double + * cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return dsdot(N, X, incX, Y, incY) * */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":85 - * cdef REAL_t f, g + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/word2vec_inner.pyx":61 + * + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return sdot(N, X, incX, Y, incY) * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelen): - * row2 = word_point[b] * size */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":86 +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_float(int const *__pyx_v_N, float const *__pyx_v_X, int const *__pyx_v_incX, float const *__pyx_v_Y, int const *__pyx_v_incY) { + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_r; + + /* "trunk/gensim/models/word2vec_inner.pyx":62 + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + * return sdot(N, X, incX, Y, incY) # <<<<<<<<<<<<<< * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelen): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * # for when no blas available */ - __pyx_t_1 = __pyx_v_codelen; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; + __pyx_r = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot(__pyx_v_N, __pyx_v_X, __pyx_v_incX, __pyx_v_Y, __pyx_v_incY)); + goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":87 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelen): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: + /* "trunk/gensim/models/word2vec_inner.pyx":61 + * + * # for when fblas.sdot returns a float + * cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * return sdot(N, X, incX, Y, incY) + * */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":88 - * for b in range(codelen): - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/word2vec_inner.pyx":65 + * + * # for when no blas available + * cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * # not a true full dot()-implementation: just enough for our cases + * cdef int i */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); - /* "trunk/gensim/models/word2vec_inner.pyx":89 - * row2 = word_point[b] * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] +static __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_noblas(int const *__pyx_v_N, float const *__pyx_v_X, CYTHON_UNUSED int const *__pyx_v_incX, float const *__pyx_v_Y, CYTHON_UNUSED int const *__pyx_v_incY) { + int __pyx_v_i; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_a; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_r; + int __pyx_t_1; + + /* "trunk/gensim/models/word2vec_inner.pyx":69 + * cdef int i + * cdef REAL_t a + * a = 0.0 # <<<<<<<<<<<<<< + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { + __pyx_v_a = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - /* "trunk/gensim/models/word2vec_inner.pyx":90 - * f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha + /* "trunk/gensim/models/word2vec_inner.pyx":70 + * cdef REAL_t a + * a = 0.0 + * for i from 0 <= i < N[0] by 1: # <<<<<<<<<<<<<< + * a += X[i] * Y[i] + * return a */ - goto __pyx_L3_continue; - } + __pyx_t_1 = (__pyx_v_N[0]); + for (__pyx_v_i = 0; __pyx_v_i < __pyx_t_1; __pyx_v_i+=1) { - /* "trunk/gensim/models/word2vec_inner.pyx":91 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":71 + * a = 0.0 + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] # <<<<<<<<<<<<<< + * return a + * */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); + __pyx_v_a = (__pyx_v_a + ((__pyx_v_X[__pyx_v_i]) * (__pyx_v_Y[__pyx_v_i]))); + } - /* "trunk/gensim/models/word2vec_inner.pyx":92 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":72 + * for i from 0 <= i < N[0] by 1: + * a += X[i] * Y[i] + * return a # <<<<<<<<<<<<<< + * + * # for when no blas available */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); + __pyx_r = __pyx_v_a; + goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":93 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":65 + * + * # for when no blas available + * cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * # not a true full dot()-implementation: just enough for our cases + * cdef int i */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":94 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "trunk/gensim/models/word2vec_inner.pyx":75 * + * # for when no blas available + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * cdef int i + * for i from 0 <= i < N[0] by 1: */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L3_continue:; - } - /* "trunk/gensim/models/word2vec_inner.pyx":95 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_saxpy_noblas(int const *__pyx_v_N, float const *__pyx_v_alpha, float const *__pyx_v_X, int const *__pyx_v_incX, float *__pyx_v_Y, int const *__pyx_v_incY) { + int __pyx_v_i; + int __pyx_t_1; + + /* "trunk/gensim/models/word2vec_inner.pyx":77 + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: + * cdef int i + * for i from 0 <= i < N[0] by 1: # <<<<<<<<<<<<<< + * Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] + * + */ + __pyx_t_1 = (__pyx_v_N[0]); + for (__pyx_v_i = 0; __pyx_v_i < __pyx_t_1; __pyx_v_i+=1) { + + /* "trunk/gensim/models/word2vec_inner.pyx":78 + * cdef int i + * for i from 0 <= i < N[0] by 1: + * Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] # <<<<<<<<<<<<<< * * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + (__pyx_v_Y[(__pyx_v_i * (__pyx_v_incY[0]))]) = (((__pyx_v_alpha[0]) * (__pyx_v_X[(__pyx_v_i * (__pyx_v_incX[0]))])) + (__pyx_v_Y[(__pyx_v_i * (__pyx_v_incY[0]))])); + } - /* "trunk/gensim/models/word2vec_inner.pyx":76 - * cdef REAL_t ONEF = 1.0 + /* "trunk/gensim/models/word2vec_inner.pyx":75 * - * cdef void fast_sentence0_sg_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * # for when no blas available + * cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: # <<<<<<<<<<<<<< + * cdef int i + * for i from 0 <= i < N[0] by 1: */ /* function exit code */ } -/* "trunk/gensim/models/word2vec_inner.pyx":98 +/* "trunk/gensim/models/word2vec_inner.pyx":81 * * - * cdef void fast_sentence1_sg_hs( # <<<<<<<<<<<<<< + * cdef void fast_sentence_sg_hs( # <<<<<<<<<<<<<< * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, * REAL_t *syn0, REAL_t *syn1, const int size, */ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work) { +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks) { PY_LONG_LONG __pyx_v_b; PY_LONG_LONG __pyx_v_row1; PY_LONG_LONG __pyx_v_row2; @@ -1546,7 +1551,7 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs int __pyx_t_3; int __pyx_t_4; - /* "trunk/gensim/models/word2vec_inner.pyx":104 + /* "trunk/gensim/models/word2vec_inner.pyx":87 * * cdef long long a, b * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< @@ -1555,7 +1560,7 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs */ __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":107 + /* "trunk/gensim/models/word2vec_inner.pyx":90 * cdef REAL_t f, g * * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< @@ -1564,38 +1569,38 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs */ memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":108 + /* "trunk/gensim/models/word2vec_inner.pyx":91 * * memset(work, 0, size * cython.sizeof(REAL_t)) * for b in range(codelen): # <<<<<<<<<<<<<< * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) */ __pyx_t_1 = __pyx_v_codelen; for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_b = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":109 + /* "trunk/gensim/models/word2vec_inner.pyx":92 * memset(work, 0, size * cython.sizeof(REAL_t)) * for b in range(codelen): * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":110 + /* "trunk/gensim/models/word2vec_inner.pyx":93 * for b in range(codelen): * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< + * f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":111 + /* "trunk/gensim/models/word2vec_inner.pyx":94 * row2 = word_point[b] * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + * f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -1611,8 +1616,8 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs __pyx_L6_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":112 - * f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":95 + * f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -1621,57 +1626,57 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs goto __pyx_L3_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":113 + /* "trunk/gensim/models/word2vec_inner.pyx":96 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/word2vec_inner.pyx":114 + /* "trunk/gensim/models/word2vec_inner.pyx":97 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) */ __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/word2vec_inner.pyx":115 + /* "trunk/gensim/models/word2vec_inner.pyx":98 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":116 + /* "trunk/gensim/models/word2vec_inner.pyx":99 * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); __pyx_L3_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":117 - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":100 + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< * * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_word_locks[__pyx_v_word2_index])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":98 + /* "trunk/gensim/models/word2vec_inner.pyx":81 * * - * cdef void fast_sentence1_sg_hs( # <<<<<<<<<<<<<< + * cdef void fast_sentence_sg_hs( # <<<<<<<<<<<<<< * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, * REAL_t *syn0, REAL_t *syn1, const int size, */ @@ -1679,2339 +1684,117 @@ static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs /* function exit code */ } -/* "trunk/gensim/models/word2vec_inner.pyx":120 +/* "trunk/gensim/models/word2vec_inner.pyx":103 * * - * cdef void fast_sentence2_sg_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, + * cdef unsigned long long fast_sentence_sg_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, + * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, */ -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int const __pyx_v_codelen, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_b; +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks) { PY_LONG_LONG __pyx_v_row1; PY_LONG_LONG __pyx_v_row2; + unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; + __pyx_t_5numpy_uint32_t __pyx_v_target_index; + int __pyx_v_d; + unsigned PY_LONG_LONG __pyx_r; + long __pyx_t_1; + int __pyx_t_2; int __pyx_t_3; - PY_LONG_LONG __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; + int __pyx_t_4; - /* "trunk/gensim/models/word2vec_inner.pyx":126 + /* "trunk/gensim/models/word2vec_inner.pyx":110 * - * cdef long long a, b + * cdef long long a * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef REAL_t f, g - * + * cdef unsigned long long modulo = 281474976710655ULL + * cdef REAL_t f, g, label */ __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":129 - * cdef REAL_t f, g - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * for b in range(codelen): + /* "trunk/gensim/models/word2vec_inner.pyx":111 + * cdef long long a + * cdef long long row1 = word2_index * size, row2 + * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< + * cdef REAL_t f, g, label + * cdef np.uint32_t target_index */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_modulo = 281474976710655ULL; - /* "trunk/gensim/models/word2vec_inner.pyx":130 + /* "trunk/gensim/models/word2vec_inner.pyx":116 + * cdef int d * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * for b in range(codelen): - * row2 = word_point[b] * size + * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< + * + * for d in range(negative+1): */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } + memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":131 - * for a in range(size): - * work[a] = 0.0 - * for b in range(codelen): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = 0.0 + /* "trunk/gensim/models/word2vec_inner.pyx":118 + * memset(work, 0, size * cython.sizeof(REAL_t)) + * + * for d in range(negative+1): # <<<<<<<<<<<<<< + * if d == 0: + * target_index = word_index */ - __pyx_t_1 = __pyx_v_codelen; + __pyx_t_1 = (__pyx_v_negative + 1); for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":132 - * work[a] = 0.0 - * for b in range(codelen): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); + __pyx_v_d = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":133 - * for b in range(codelen): - * row2 = word_point[b] * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] + /* "trunk/gensim/models/word2vec_inner.pyx":119 + * + * for d in range(negative+1): + * if d == 0: # <<<<<<<<<<<<<< + * target_index = word_index + * label = ONEF */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); + __pyx_t_3 = ((__pyx_v_d == 0) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":134 - * row2 = word_point[b] * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: + /* "trunk/gensim/models/word2vec_inner.pyx":120 + * for d in range(negative+1): + * if d == 0: + * target_index = word_index # <<<<<<<<<<<<<< + * label = ONEF + * else: */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; + __pyx_v_target_index = __pyx_v_word_index; - /* "trunk/gensim/models/word2vec_inner.pyx":135 - * f = 0.0 - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue + /* "trunk/gensim/models/word2vec_inner.pyx":121 + * if d == 0: + * target_index = word_index + * label = ONEF # <<<<<<<<<<<<<< + * else: + * target_index = table[(next_random >> 16) % table_len] */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]) * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); + __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; + goto __pyx_L5; } + /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":136 - * for a in range(size): - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] + /* "trunk/gensim/models/word2vec_inner.pyx":123 + * label = ONEF + * else: + * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< + * next_random = (next_random * 25214903917ULL + 11) & modulo + * if target_index == word_index: */ - __pyx_t_6 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_6) { - } else { - __pyx_t_5 = __pyx_t_6; - goto __pyx_L10_bool_binop_done; - } - __pyx_t_6 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_5 = __pyx_t_6; - __pyx_L10_bool_binop_done:; - if (__pyx_t_5) { + __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - /* "trunk/gensim/models/word2vec_inner.pyx":137 - * f += syn0[row1 + a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha + /* "trunk/gensim/models/word2vec_inner.pyx":124 + * else: + * target_index = table[(next_random >> 16) % table_len] + * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< + * if target_index == word_index: + * continue */ - goto __pyx_L5_continue; - } + __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - /* "trunk/gensim/models/word2vec_inner.pyx":138 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":139 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1[row2 + a] - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":140 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1[row2 + a] - * for a in range(size): - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; - - /* "trunk/gensim/models/word2vec_inner.pyx":141 - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - * work[a] += g * syn1[row2 + a] # <<<<<<<<<<<<<< - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":142 - * for a in range(size): - * work[a] += g * syn1[row2 + a] - * for a in range(size): # <<<<<<<<<<<<<< - * syn1[row2 + a] += g * syn0[row1 + a] - * for a in range(size): - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_v_a = __pyx_t_4; - - /* "trunk/gensim/models/word2vec_inner.pyx":143 - * work[a] += g * syn1[row2 + a] - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] # <<<<<<<<<<<<<< - * for a in range(size): - * syn0[row1 + a] += work[a] - */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1[__pyx_t_7]) = ((__pyx_v_syn1[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]))); - } - __pyx_L5_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":144 - * for a in range(size): - * syn1[row2 + a] += g * syn0[row1 + a] - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[row1 + a] += work[a] - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":145 - * syn1[row2 + a] += g * syn0[row1 + a] - * for a in range(size): - * syn0[row1 + a] += work[a] # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_4 = (__pyx_v_row1 + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_4]) = ((__pyx_v_syn0[__pyx_t_4]) + (__pyx_v_work[__pyx_v_a])); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":120 - * - * - * cdef void fast_sentence2_sg_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - * REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/word2vec_inner.pyx":148 - * - * - * cdef unsigned long long fast_sentence0_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random) { - PY_LONG_LONG __pyx_v_row1; - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - int __pyx_v_d; - unsigned PY_LONG_LONG __pyx_r; - long __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - - /* "trunk/gensim/models/word2vec_inner.pyx":155 - * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":156 - * cdef long long a - * cdef long long row1 = word2_index * size, row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, label - * cdef np.uint32_t target_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/word2vec_inner.pyx":161 - * cdef int d - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":163 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index - */ - __pyx_t_1 = (__pyx_v_negative + 1); - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_d = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":164 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":165 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/word2vec_inner.pyx":166 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L5; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":168 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/word2vec_inner.pyx":169 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/word2vec_inner.pyx":170 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":171 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L3_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":172 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - __pyx_L5:; - - /* "trunk/gensim/models/word2vec_inner.pyx":174 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":175 - * - * row2 = target_index * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); - - /* "trunk/gensim/models/word2vec_inner.pyx":176 - * row2 = target_index * size - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L8_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":177 - * f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L3_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":178 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":179 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":180 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":181 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":183 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":185 - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence1_sg_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/word2vec_inner.pyx":148 - * - * - * cdef unsigned long long fast_sentence0_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - - /* function exit code */ - __pyx_L0:; - return __pyx_r; -} - -/* "trunk/gensim/models/word2vec_inner.pyx":187 - * return next_random - * - * cdef unsigned long long fast_sentence1_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random) { - PY_LONG_LONG __pyx_v_row1; - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - int __pyx_v_d; - unsigned PY_LONG_LONG __pyx_r; - long __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - - /* "trunk/gensim/models/word2vec_inner.pyx":194 - * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":195 - * cdef long long a - * cdef long long row1 = word2_index * size, row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, label - * cdef np.uint32_t target_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/word2vec_inner.pyx":200 - * cdef int d - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":202 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * - * if d == 0: - */ - __pyx_t_1 = (__pyx_v_negative + 1); - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_d = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":204 - * for d in range(negative+1): - * - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":205 - * - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/word2vec_inner.pyx":206 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L5; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":208 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/word2vec_inner.pyx":209 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/word2vec_inner.pyx":210 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":211 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L3_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":212 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - __pyx_L5:; - - /* "trunk/gensim/models/word2vec_inner.pyx":214 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":215 - * - * row2 = target_index * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); - - /* "trunk/gensim/models/word2vec_inner.pyx":216 - * row2 = target_index * size - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L8_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":217 - * f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L3_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":218 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":219 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":220 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":221 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< - * - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":223 - * saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - * - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":225 - * saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence2_sg_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/word2vec_inner.pyx":187 - * return next_random - * - * cdef unsigned long long fast_sentence1_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - - /* function exit code */ - __pyx_L0:; - return __pyx_r; -} - -/* "trunk/gensim/models/word2vec_inner.pyx":227 - * return next_random - * - * cdef unsigned long long fast_sentence2_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const __pyx_v_word_index, __pyx_t_5numpy_uint32_t const __pyx_v_word2_index, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, unsigned PY_LONG_LONG __pyx_v_next_random) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_row1; - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - int __pyx_v_d; - unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - long __pyx_t_3; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - - /* "trunk/gensim/models/word2vec_inner.pyx":234 - * - * cdef long long a - * cdef long long row1 = word2_index * size, row2 # <<<<<<<<<<<<<< - * cdef unsigned long long modulo = 281474976710655ULL - * cdef REAL_t f, g, label - */ - __pyx_v_row1 = (__pyx_v_word2_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":235 - * cdef long long a - * cdef long long row1 = word2_index * size, row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, label - * cdef np.uint32_t target_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/word2vec_inner.pyx":240 - * cdef int d - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":241 - * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":243 - * work[a] = 0.0 - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * - * if d == 0: - */ - __pyx_t_3 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_3; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; - - /* "trunk/gensim/models/word2vec_inner.pyx":245 - * for d in range(negative+1): - * - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_4 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":246 - * - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/word2vec_inner.pyx":247 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L7; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":249 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/word2vec_inner.pyx":250 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/word2vec_inner.pyx":251 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_4 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":252 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L5_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":253 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< - * - * row2 = target_index * size - */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - __pyx_L7:; - - /* "trunk/gensim/models/word2vec_inner.pyx":255 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":256 - * - * row2 = target_index * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":257 - * row2 = target_index * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += syn0[row1 + a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":258 - * f = 0.0 - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]) * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":259 - * for a in range(size): - * f += syn0[row1 + a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_6 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_6) { - } else { - __pyx_t_4 = __pyx_t_6; - goto __pyx_L12_bool_binop_done; - } - __pyx_t_6 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_6; - __pyx_L12_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":260 - * f += syn0[row1 + a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - */ - goto __pyx_L5_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":261 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * for a in range(size): - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":262 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":263 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): - */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":264 - * g = (label - f) * alpha - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * for a in range(size): - * syn1neg[row2 + a] += g * syn0[row1 + a] - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":265 - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): # <<<<<<<<<<<<<< - * syn1neg[row2 + a] += g * syn0[row1 + a] - * - */ - __pyx_t_5 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_5; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":266 - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): - * syn1neg[row2 + a] += g * syn0[row1 + a] # <<<<<<<<<<<<<< - * - * for a in range(size): - */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1neg[__pyx_t_7]) = ((__pyx_v_syn1neg[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn0[(__pyx_v_row1 + __pyx_v_a)]))); - } - __pyx_L5_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":268 - * syn1neg[row2 + a] += g * syn0[row1 + a] - * - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[row1 + a] += work[a] - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":269 - * - * for a in range(size): - * syn0[row1 + a] += work[a] # <<<<<<<<<<<<<< - * - * return next_random - */ - __pyx_t_7 = (__pyx_v_row1 + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":271 - * syn0[row1 + a] += work[a] - * - * return next_random # <<<<<<<<<<<<<< - * - * cdef void fast_sentence0_cbow_hs( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/word2vec_inner.pyx":227 - * return next_random - * - * cdef unsigned long long fast_sentence2_sg_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, - * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - */ - - /* function exit code */ - __pyx_L0:; - return __pyx_r; -} - -/* "trunk/gensim/models/word2vec_inner.pyx":273 - * return next_random - * - * cdef void fast_sentence0_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_inv_count; - int __pyx_v_m; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - PY_LONG_LONG __pyx_t_5; - - /* "trunk/gensim/models/word2vec_inner.pyx":284 - * cdef int m - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":285 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":286 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":287 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":288 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":290 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - - /* "trunk/gensim/models/word2vec_inner.pyx":291 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":292 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) - */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L9_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L9_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":293 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/word2vec_inner.pyx":294 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - goto __pyx_L8; - } - __pyx_L8:; - - /* "trunk/gensim/models/word2vec_inner.pyx":296 - * sscal(&size, &inv_count, neu1, &ONE) - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":297 - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_1; __pyx_t_5+=1) { - __pyx_v_b = __pyx_t_5; - - /* "trunk/gensim/models/word2vec_inner.pyx":298 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":299 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); - - /* "trunk/gensim/models/word2vec_inner.pyx":300 - * row2 = word_point[b] * size - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L14_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L14_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":301 - * f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L11_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":302 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":303 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":304 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":305 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * - * for m in range(j, k): - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L11_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":307 - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":308 - * - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L19_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L19_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":309 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - */ - goto __pyx_L16_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":311 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) # <<<<<<<<<<<<<< - * - * cdef void fast_sentence1_cbow_hs( - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L16_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":273 - * return next_random - * - * cdef void fast_sentence0_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/word2vec_inner.pyx":313 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - * - * cdef void fast_sentence1_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean) { - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_inv_count; - int __pyx_v_m; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - PY_LONG_LONG __pyx_t_5; - - /* "trunk/gensim/models/word2vec_inner.pyx":324 - * cdef int m - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":325 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":326 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":327 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":328 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":330 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - - /* "trunk/gensim/models/word2vec_inner.pyx":331 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":332 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count , neu1, &ONE) - */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L9_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L9_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":333 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count , neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/word2vec_inner.pyx":334 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count , neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - goto __pyx_L8; - } - __pyx_L8:; - - /* "trunk/gensim/models/word2vec_inner.pyx":336 - * sscal(&size, &inv_count , neu1, &ONE) - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":337 - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_1; __pyx_t_5+=1) { - __pyx_v_b = __pyx_t_5; - - /* "trunk/gensim/models/word2vec_inner.pyx":338 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":339 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); - - /* "trunk/gensim/models/word2vec_inner.pyx":340 - * row2 = word_point[b] * size - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L14_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L14_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":341 - * f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L11_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":342 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":343 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":344 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - - /* "trunk/gensim/models/word2vec_inner.pyx":345 - * g = (1 - word_code[b] - f) * alpha - * saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< - * - * for m in range(j, k): - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L11_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":347 - * saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - * - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":348 - * - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L19_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L19_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":349 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - */ - goto __pyx_L16_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":351 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< - * - * cdef void fast_sentence2_cbow_hs( - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L16_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":313 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - * - * cdef void fast_sentence1_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/word2vec_inner.pyx":353 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * - * cdef void fast_sentence2_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - -static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean) { - PY_LONG_LONG __pyx_v_a; - PY_LONG_LONG __pyx_v_b; - PY_LONG_LONG __pyx_v_row2; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; - int __pyx_v_m; - int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - PY_LONG_LONG __pyx_t_8; - - /* "trunk/gensim/models/word2vec_inner.pyx":364 - * cdef int m - * - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] = 0.0 - * count = 0.0 - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":365 - * - * for a in range(size): - * neu1[a] = 0.0 # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - (__pyx_v_neu1[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":366 - * for a in range(size): - * neu1[a] = 0.0 - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":367 - * neu1[a] = 0.0 - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/word2vec_inner.pyx":368 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L8_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L8_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":369 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L5_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":371 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - - /* "trunk/gensim/models/word2vec_inner.pyx":372 - * else: - * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":373 - * count += ONEF - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * for a in range(size): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L5_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":374 - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] /= count - */ - __pyx_t_5 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L13_bool_binop_done; - } - __pyx_t_5 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L13_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":375 - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] /= count - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":376 - * if cbow_mean and count > (0.5): - * for a in range(size): - * neu1[a] /= count # <<<<<<<<<<<<<< - * - * for a in range(size): - */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) / __pyx_v_count); - } - goto __pyx_L12; - } - __pyx_L12:; - - /* "trunk/gensim/models/word2vec_inner.pyx":378 - * neu1[a] /= count - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * for b in range(codelens[i]): - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":379 - * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< - * for b in range(codelens[i]): - * row2 = word_point[b] * size - */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":380 - * for a in range(size): - * work[a] = 0.0 - * for b in range(codelens[i]): # <<<<<<<<<<<<<< - * row2 = word_point[b] * size - * f = 0.0 - */ - __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_b = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":381 - * work[a] = 0.0 - * for b in range(codelens[i]): - * row2 = word_point[b] * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): - */ - __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - - /* "trunk/gensim/models/word2vec_inner.pyx":382 - * for b in range(codelens[i]): - * row2 = word_point[b] * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":383 - * row2 = word_point[b] * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/word2vec_inner.pyx":384 - * f = 0.0 - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] # <<<<<<<<<<<<<< - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_neu1[__pyx_v_a]) * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":385 - * for a in range(size): - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - */ - __pyx_t_5 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L24_bool_binop_done; - } - __pyx_t_5 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L24_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":386 - * f += neu1[a] * syn1[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue # <<<<<<<<<<<<<< - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - */ - goto __pyx_L19_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":387 - * if f <= -MAX_EXP or f >= MAX_EXP: - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - */ - __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - - /* "trunk/gensim/models/word2vec_inner.pyx":388 - * continue - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1[row2 + a] - */ - __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - - /* "trunk/gensim/models/word2vec_inner.pyx":389 - * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1[row2 + a] - * for a in range(size): - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/word2vec_inner.pyx":390 - * g = (1 - word_code[b] - f) * alpha - * for a in range(size): - * work[a] += g * syn1[row2 + a] # <<<<<<<<<<<<<< - * for a in range(size): - * syn1[row2 + a] += g * neu1[a] - */ - __pyx_t_8 = __pyx_v_a; - (__pyx_v_work[__pyx_t_8]) = ((__pyx_v_work[__pyx_t_8]) + (__pyx_v_g * (__pyx_v_syn1[(__pyx_v_row2 + __pyx_v_a)]))); - } - - /* "trunk/gensim/models/word2vec_inner.pyx":391 - * for a in range(size): - * work[a] += g * syn1[row2 + a] - * for a in range(size): # <<<<<<<<<<<<<< - * syn1[row2 + a] += g * neu1[a] - * - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_3; __pyx_t_7+=1) { - __pyx_v_a = __pyx_t_7; - - /* "trunk/gensim/models/word2vec_inner.pyx":392 - * work[a] += g * syn1[row2 + a] - * for a in range(size): - * syn1[row2 + a] += g * neu1[a] # <<<<<<<<<<<<<< - * - * for m in range(j, k): - */ - __pyx_t_8 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1[__pyx_t_8]) = ((__pyx_v_syn1[__pyx_t_8]) + (__pyx_v_g * (__pyx_v_neu1[__pyx_v_a]))); - } - __pyx_L19_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":394 - * syn1[row2 + a] += g * neu1[a] - * - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; - - /* "trunk/gensim/models/word2vec_inner.pyx":395 - * - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L33_bool_binop_done; - } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L33_bool_binop_done:; - if (__pyx_t_4) { - - /* "trunk/gensim/models/word2vec_inner.pyx":396 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * for a in range(size): - */ - goto __pyx_L30_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":398 - * continue - * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[indexes[m] * size + a] += work[a] - * - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":399 - * else: - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< - * - * cdef unsigned long long fast_sentence0_cbow_neg( - */ - __pyx_t_7 = (((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } - } - __pyx_L30_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":353 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * - * cdef void fast_sentence2_cbow_hs( # <<<<<<<<<<<<<< - * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - */ - - /* function exit code */ -} - -/* "trunk/gensim/models/word2vec_inner.pyx":401 - * syn0[indexes[m] * size + a] += work[a] - * - * cdef unsigned long long fast_sentence0_cbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - */ - -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random) { - PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_inv_count; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - __pyx_t_5numpy_uint32_t __pyx_v_word_index; - int __pyx_v_d; - int __pyx_v_m; - unsigned PY_LONG_LONG __pyx_r; - int __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_t_4; - long __pyx_t_5; - - /* "trunk/gensim/models/word2vec_inner.pyx":409 - * cdef long long a - * cdef long long row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label - * cdef np.uint32_t target_index, word_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/word2vec_inner.pyx":414 - * cdef int d, m - * - * word_index = indexes[i] # <<<<<<<<<<<<<< - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); - - /* "trunk/gensim/models/word2vec_inner.pyx":416 - * word_index = indexes[i] - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * count = 0.0 - * for m in range(j, k): - */ - memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":417 - * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 # <<<<<<<<<<<<<< - * for m in range(j, k): - * if m == i or codelens[m] == 0: - */ - __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":418 - * memset(neu1, 0, size * cython.sizeof(REAL_t)) - * count = 0.0 - * for m in range(j, k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":419 - * count = 0.0 - * for m in range(j, k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L6_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L6_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":420 - * for m in range(j, k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * count += ONEF - */ - goto __pyx_L3_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":422 - * continue - * else: - * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - */ - __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - - /* "trunk/gensim/models/word2vec_inner.pyx":423 - * else: - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L3_continue:; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":424 - * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) - */ - __pyx_t_4 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L9_bool_binop_done; - } - __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L9_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":425 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/word2vec_inner.pyx":426 - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< - * - * memset(work, 0, size * cython.sizeof(REAL_t)) - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - goto __pyx_L8; - } - __pyx_L8:; - - /* "trunk/gensim/models/word2vec_inner.pyx":428 - * sscal(&size, &inv_count, neu1, &ONE) - * - * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): - */ - memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - - /* "trunk/gensim/models/word2vec_inner.pyx":430 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index - */ - __pyx_t_5 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_5; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; - - /* "trunk/gensim/models/word2vec_inner.pyx":431 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":432 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/word2vec_inner.pyx":433 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L13; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":435 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/word2vec_inner.pyx":436 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/word2vec_inner.pyx":437 + /* "trunk/gensim/models/word2vec_inner.pyx":125 * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo * if target_index == word_index: # <<<<<<<<<<<<<< @@ -4021,17 +1804,17 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":438 + /* "trunk/gensim/models/word2vec_inner.pyx":126 * next_random = (next_random * 25214903917ULL + 11) & modulo * if target_index == word_index: * continue # <<<<<<<<<<<<<< * label = 0.0 * */ - goto __pyx_L11_continue; + goto __pyx_L3_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":439 + /* "trunk/gensim/models/word2vec_inner.pyx":127 * if target_index == word_index: * continue * label = 0.0 # <<<<<<<<<<<<<< @@ -4040,29 +1823,29 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); } - __pyx_L13:; + __pyx_L5:; - /* "trunk/gensim/models/word2vec_inner.pyx":441 + /* "trunk/gensim/models/word2vec_inner.pyx":129 * label = 0.0 * * row2 = target_index * size # <<<<<<<<<<<<<< - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + * f = our_dot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":442 + /* "trunk/gensim/models/word2vec_inner.pyx":130 * * row2 = target_index * size - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * f = our_dot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot((&__pyx_v_size), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":443 + /* "trunk/gensim/models/word2vec_inner.pyx":131 * row2 = target_index * size - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + * f = our_dot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -4071,129 +1854,86 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas if (!__pyx_t_4) { } else { __pyx_t_3 = __pyx_t_4; - goto __pyx_L16_bool_binop_done; + goto __pyx_L8_bool_binop_done; } __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); __pyx_t_3 = __pyx_t_4; - __pyx_L16_bool_binop_done:; + __pyx_L8_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":444 - * f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":132 + * f = our_dot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha */ - goto __pyx_L11_continue; + goto __pyx_L3_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":445 + /* "trunk/gensim/models/word2vec_inner.pyx":133 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/word2vec_inner.pyx":446 + /* "trunk/gensim/models/word2vec_inner.pyx":134 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) */ __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/word2vec_inner.pyx":447 + /* "trunk/gensim/models/word2vec_inner.pyx":135 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":448 + /* "trunk/gensim/models/word2vec_inner.pyx":136 * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< * - * for m in range(j,k): + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L11_continue:; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_L3_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":450 - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - * - * for m in range(j,k): # <<<<<<<<<<<<<< - * if m == i or codelens[m] == 0: - * continue - */ - __pyx_t_1 = __pyx_v_k; - for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_m = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":451 + /* "trunk/gensim/models/word2vec_inner.pyx":138 + * our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) * - * for m in range(j,k): - * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< - * continue - * else: - */ - __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_4) { - } else { - __pyx_t_3 = __pyx_t_4; - goto __pyx_L21_bool_binop_done; - } - __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_3 = __pyx_t_4; - __pyx_L21_bool_binop_done:; - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":452 - * for m in range(j,k): - * if m == i or codelens[m] == 0: - * continue # <<<<<<<<<<<<<< - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - */ - goto __pyx_L18_continue; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":454 - * continue - * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) # <<<<<<<<<<<<<< * * return next_random */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - } - __pyx_L18_continue:; - } + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_word_locks[__pyx_v_word2_index])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[__pyx_v_row1])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":456 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":140 + * our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) * * return next_random # <<<<<<<<<<<<<< * - * cdef unsigned long long fast_sentence1_cbow_neg( + * */ __pyx_r = __pyx_v_next_random; goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":401 - * syn0[indexes[m] * size + a] += work[a] + /* "trunk/gensim/models/word2vec_inner.pyx":103 * - * cdef unsigned long long fast_sentence0_cbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * + * cdef unsigned long long fast_sentence_sg_neg( # <<<<<<<<<<<<<< + * const int negative, np.uint32_t *table, unsigned long long table_len, + * REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, */ /* function exit code */ @@ -4201,53 +1941,39 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas return __pyx_r; } -/* "trunk/gensim/models/word2vec_inner.pyx":458 - * return next_random +/* "trunk/gensim/models/word2vec_inner.pyx":143 * - * cdef unsigned long long fast_sentence1_cbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * + * cdef void fast_sentence_cbow_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], + * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, */ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random) { +static void __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs(__pyx_t_5numpy_uint32_t const *__pyx_v_word_point, __pyx_t_5numpy_uint8_t const *__pyx_v_word_code, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks) { + PY_LONG_LONG __pyx_v_b; PY_LONG_LONG __pyx_v_row2; - unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_inv_count; - __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; - __pyx_t_5numpy_uint32_t __pyx_v_target_index; - __pyx_t_5numpy_uint32_t __pyx_v_word_index; - int __pyx_v_d; int __pyx_v_m; - unsigned PY_LONG_LONG __pyx_r; int __pyx_t_1; int __pyx_t_2; int __pyx_t_3; int __pyx_t_4; - long __pyx_t_5; + PY_LONG_LONG __pyx_t_5; - /* "trunk/gensim/models/word2vec_inner.pyx":466 - * cdef long long a + /* "trunk/gensim/models/word2vec_inner.pyx":151 + * cdef long long a, b * cdef long long row2 - * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label - * cdef np.uint32_t target_index, word_index - */ - __pyx_v_modulo = 281474976710655ULL; - - /* "trunk/gensim/models/word2vec_inner.pyx":471 - * cdef int d, m - * - * word_index = indexes[i] # <<<<<<<<<<<<<< + * cdef REAL_t f, g, count, inv_count = 1.0 # <<<<<<<<<<<<<< + * cdef int m * - * memset(neu1, 0, size * cython.sizeof(REAL_t)) */ - __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); + __pyx_v_inv_count = 1.0; - /* "trunk/gensim/models/word2vec_inner.pyx":473 - * word_index = indexes[i] + /* "trunk/gensim/models/word2vec_inner.pyx":154 + * cdef int m * * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< * count = 0.0 @@ -4255,7 +1981,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":474 + /* "trunk/gensim/models/word2vec_inner.pyx":155 * * memset(neu1, 0, size * cython.sizeof(REAL_t)) * count = 0.0 # <<<<<<<<<<<<<< @@ -4264,7 +1990,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - /* "trunk/gensim/models/word2vec_inner.pyx":475 + /* "trunk/gensim/models/word2vec_inner.pyx":156 * memset(neu1, 0, size * cython.sizeof(REAL_t)) * count = 0.0 * for m in range(j, k): # <<<<<<<<<<<<<< @@ -4275,7 +2001,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_m = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":476 + /* "trunk/gensim/models/word2vec_inner.pyx":157 * count = 0.0 * for m in range(j, k): * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< @@ -4293,7 +2019,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas __pyx_L6_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":477 + /* "trunk/gensim/models/word2vec_inner.pyx":158 * for m in range(j, k): * if m == i or codelens[m] == 0: * continue # <<<<<<<<<<<<<< @@ -4304,187 +2030,120 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":479 + /* "trunk/gensim/models/word2vec_inner.pyx":160 * continue * else: * count += ONEF # <<<<<<<<<<<<<< - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): */ __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - /* "trunk/gensim/models/word2vec_inner.pyx":480 + /* "trunk/gensim/models/word2vec_inner.pyx":161 * else: * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< + * if count > (0.5): * inv_count = ONEF/count */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); } __pyx_L3_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":481 + /* "trunk/gensim/models/word2vec_inner.pyx":162 * count += ONEF - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): # <<<<<<<<<<<<<< + * inv_count = ONEF/count + * if cbow_mean and count > (0.5): + */ + __pyx_t_3 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); + if (__pyx_t_3) { + + /* "trunk/gensim/models/word2vec_inner.pyx":163 + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): + * inv_count = ONEF/count # <<<<<<<<<<<<<< + * if cbow_mean and count > (0.5): + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) + */ + __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); + goto __pyx_L8; + } + __pyx_L8:; + + /* "trunk/gensim/models/word2vec_inner.pyx":164 + * if count > (0.5): * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) + * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) + * */ __pyx_t_4 = (__pyx_v_cbow_mean != 0); if (__pyx_t_4) { } else { __pyx_t_3 = __pyx_t_4; - goto __pyx_L9_bool_binop_done; + goto __pyx_L10_bool_binop_done; } __pyx_t_4 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); __pyx_t_3 = __pyx_t_4; - __pyx_L9_bool_binop_done:; + __pyx_L10_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":482 - * saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - * if cbow_mean and count > (0.5): - * inv_count = ONEF/count # <<<<<<<<<<<<<< - * sscal(&size, &inv_count, neu1, &ONE) - * - */ - __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); - - /* "trunk/gensim/models/word2vec_inner.pyx":483 - * if cbow_mean and count > (0.5): + /* "trunk/gensim/models/word2vec_inner.pyx":165 * inv_count = ONEF/count - * sscal(&size, &inv_count, neu1, &ONE) # <<<<<<<<<<<<<< + * if cbow_mean and count > (0.5): + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< * * memset(work, 0, size * cython.sizeof(REAL_t)) */ __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - goto __pyx_L8; + goto __pyx_L9; } - __pyx_L8:; + __pyx_L9:; - /* "trunk/gensim/models/word2vec_inner.pyx":485 - * sscal(&size, &inv_count, neu1, &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":167 + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) * * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< - * - * for d in range(negative+1): + * for b in range(codelens[i]): + * row2 = word_point[b] * size */ memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":487 - * memset(work, 0, size * cython.sizeof(REAL_t)) - * - * for d in range(negative+1): # <<<<<<<<<<<<<< - * if d == 0: - * target_index = word_index - */ - __pyx_t_5 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_5; __pyx_t_1+=1) { - __pyx_v_d = __pyx_t_1; - - /* "trunk/gensim/models/word2vec_inner.pyx":488 - * - * for d in range(negative+1): - * if d == 0: # <<<<<<<<<<<<<< - * target_index = word_index - * label = ONEF - */ - __pyx_t_3 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":489 - * for d in range(negative+1): - * if d == 0: - * target_index = word_index # <<<<<<<<<<<<<< - * label = ONEF - * else: - */ - __pyx_v_target_index = __pyx_v_word_index; - - /* "trunk/gensim/models/word2vec_inner.pyx":490 - * if d == 0: - * target_index = word_index - * label = ONEF # <<<<<<<<<<<<<< - * else: - * target_index = table[(next_random >> 16) % table_len] - */ - __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L13; - } - /*else*/ { - - /* "trunk/gensim/models/word2vec_inner.pyx":492 - * label = ONEF - * else: - * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - */ - __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - - /* "trunk/gensim/models/word2vec_inner.pyx":493 - * else: - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< - * if target_index == word_index: - * continue - */ - __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - - /* "trunk/gensim/models/word2vec_inner.pyx":494 - * target_index = table[(next_random >> 16) % table_len] - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: # <<<<<<<<<<<<<< - * continue - * label = 0.0 - */ - __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_3) { - - /* "trunk/gensim/models/word2vec_inner.pyx":495 - * next_random = (next_random * 25214903917ULL + 11) & modulo - * if target_index == word_index: - * continue # <<<<<<<<<<<<<< - * label = 0.0 - * - */ - goto __pyx_L11_continue; - } - - /* "trunk/gensim/models/word2vec_inner.pyx":496 - * if target_index == word_index: - * continue - * label = 0.0 # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":168 * - * row2 = target_index * size + * memset(work, 0, size * cython.sizeof(REAL_t)) + * for b in range(codelens[i]): # <<<<<<<<<<<<<< + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) */ - __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } - __pyx_L13:; + __pyx_t_1 = (__pyx_v_codelens[__pyx_v_i]); + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_1; __pyx_t_5+=1) { + __pyx_v_b = __pyx_t_5; - /* "trunk/gensim/models/word2vec_inner.pyx":498 - * label = 0.0 - * - * row2 = target_index * size # <<<<<<<<<<<<<< - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":169 + * memset(work, 0, size * cython.sizeof(REAL_t)) + * for b in range(codelens[i]): + * row2 = word_point[b] * size # <<<<<<<<<<<<<< + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: */ - __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); + __pyx_v_row2 = ((__pyx_v_word_point[__pyx_v_b]) * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":499 - * - * row2 = target_index * size - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":170 + * for b in range(codelens[i]): + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)__pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE))); + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":500 - * row2 = target_index * size - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":171 + * row2 = word_point[b] * size + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] @@ -4493,65 +2152,87 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas if (!__pyx_t_4) { } else { __pyx_t_3 = __pyx_t_4; - goto __pyx_L16_bool_binop_done; + goto __pyx_L15_bool_binop_done; } __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); __pyx_t_3 = __pyx_t_4; - __pyx_L16_bool_binop_done:; + __pyx_L15_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":501 - * f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":172 + * f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha + * g = (1 - word_code[b] - f) * alpha */ - goto __pyx_L11_continue; + goto __pyx_L12_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":502 + /* "trunk/gensim/models/word2vec_inner.pyx":173 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/word2vec_inner.pyx":503 + /* "trunk/gensim/models/word2vec_inner.pyx":174 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha # <<<<<<<<<<<<<< - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * g = (1 - word_code[b] - f) * alpha # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) */ - __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); + __pyx_v_g = (((1 - (__pyx_v_word_code[__pyx_v_b])) - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/word2vec_inner.pyx":504 + /* "trunk/gensim/models/word2vec_inner.pyx":175 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":505 - * g = (label - f) * alpha - * saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":176 + * g = (1 - word_code[b] - f) * alpha + * our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) # <<<<<<<<<<<<<< * - * for m in range(j,k): + * if not cbow_mean: # divide error over summed window vectors */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - __pyx_L11_continue:; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_L12_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":507 - * saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + /* "trunk/gensim/models/word2vec_inner.pyx":178 + * our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) * - * for m in range(j,k): # <<<<<<<<<<<<<< + * if not cbow_mean: # divide error over summed window vectors # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) + * + */ + __pyx_t_3 = ((!(__pyx_v_cbow_mean != 0)) != 0); + if (__pyx_t_3) { + + /* "trunk/gensim/models/word2vec_inner.pyx":179 + * + * if not cbow_mean: # divide error over summed window vectors + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< + * + * for m in range(j, k): + */ + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + goto __pyx_L17; + } + __pyx_L17:; + + /* "trunk/gensim/models/word2vec_inner.pyx":181 + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) + * + * for m in range(j, k): # <<<<<<<<<<<<<< * if m == i or codelens[m] == 0: * continue */ @@ -4559,9 +2240,9 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_m = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":508 + /* "trunk/gensim/models/word2vec_inner.pyx":182 * - * for m in range(j,k): + * for m in range(j, k): * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< * continue * else: @@ -4577,67 +2258,55 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas __pyx_L21_bool_binop_done:; if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":509 - * for m in range(j,k): + /* "trunk/gensim/models/word2vec_inner.pyx":183 + * for m in range(j, k): * if m == i or codelens[m] == 0: * continue # <<<<<<<<<<<<<< * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) + * our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m] * size], &ONE) */ goto __pyx_L18_continue; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":511 + /* "trunk/gensim/models/word2vec_inner.pyx":185 * continue * else: - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m] * size], &ONE) # <<<<<<<<<<<<<< + * * - * return next_random */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_word_locks[(__pyx_v_indexes[__pyx_v_m])])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); } __pyx_L18_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":513 - * saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - * - * return next_random # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":143 * - * cdef unsigned long long fast_sentence2_cbow_neg( - */ - __pyx_r = __pyx_v_next_random; - goto __pyx_L0; - - /* "trunk/gensim/models/word2vec_inner.pyx":458 - * return next_random * - * cdef unsigned long long fast_sentence1_cbow_neg( # <<<<<<<<<<<<<< - * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, + * cdef void fast_sentence_cbow_hs( # <<<<<<<<<<<<<< + * const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], + * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, */ /* function exit code */ - __pyx_L0:; - return __pyx_r; } -/* "trunk/gensim/models/word2vec_inner.pyx":515 - * return next_random +/* "trunk/gensim/models/word2vec_inner.pyx":188 * - * cdef unsigned long long fast_sentence2_cbow_neg( # <<<<<<<<<<<<<< + * + * cdef unsigned long long fast_sentence_cbow_neg( # <<<<<<<<<<<<<< * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, */ -static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random) { - PY_LONG_LONG __pyx_v_a; +static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg(int const __pyx_v_negative, __pyx_t_5numpy_uint32_t *__pyx_v_table, unsigned PY_LONG_LONG __pyx_v_table_len, int *__pyx_v_codelens, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn1neg, int const __pyx_v_size, __pyx_t_5numpy_uint32_t const *__pyx_v_indexes, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t const __pyx_v_alpha, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work, int __pyx_v_i, int __pyx_v_j, int __pyx_v_k, int __pyx_v_cbow_mean, unsigned PY_LONG_LONG __pyx_v_next_random, __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks) { PY_LONG_LONG __pyx_v_row2; unsigned PY_LONG_LONG __pyx_v_modulo; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_f; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_g; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_count; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_inv_count; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v_label; __pyx_t_5numpy_uint32_t __pyx_v_target_index; __pyx_t_5numpy_uint32_t __pyx_v_word_index; @@ -4645,222 +2314,192 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas int __pyx_v_m; unsigned PY_LONG_LONG __pyx_r; int __pyx_t_1; - PY_LONG_LONG __pyx_t_2; + int __pyx_t_2; int __pyx_t_3; int __pyx_t_4; - int __pyx_t_5; - int __pyx_t_6; - PY_LONG_LONG __pyx_t_7; - long __pyx_t_8; + long __pyx_t_5; - /* "trunk/gensim/models/word2vec_inner.pyx":523 + /* "trunk/gensim/models/word2vec_inner.pyx":196 * cdef long long a * cdef long long row2 * cdef unsigned long long modulo = 281474976710655ULL # <<<<<<<<<<<<<< - * cdef REAL_t f, g, count, inv_count, label + * cdef REAL_t f, g, count, inv_count = 1.0, label * cdef np.uint32_t target_index, word_index */ __pyx_v_modulo = 281474976710655ULL; - /* "trunk/gensim/models/word2vec_inner.pyx":528 + /* "trunk/gensim/models/word2vec_inner.pyx":197 + * cdef long long row2 + * cdef unsigned long long modulo = 281474976710655ULL + * cdef REAL_t f, g, count, inv_count = 1.0, label # <<<<<<<<<<<<<< + * cdef np.uint32_t target_index, word_index + * cdef int d, m + */ + __pyx_v_inv_count = 1.0; + + /* "trunk/gensim/models/word2vec_inner.pyx":201 * cdef int d, m * * word_index = indexes[i] # <<<<<<<<<<<<<< * - * for a in range(size): + * memset(neu1, 0, size * cython.sizeof(REAL_t)) */ __pyx_v_word_index = (__pyx_v_indexes[__pyx_v_i]); - /* "trunk/gensim/models/word2vec_inner.pyx":530 + /* "trunk/gensim/models/word2vec_inner.pyx":203 * word_index = indexes[i] * - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] = 0.0 - * count = 0.0 - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":531 - * - * for a in range(size): - * neu1[a] = 0.0 # <<<<<<<<<<<<<< + * memset(neu1, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< * count = 0.0 * for m in range(j, k): */ - (__pyx_v_neu1[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } + memset(__pyx_v_neu1, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":532 - * for a in range(size): - * neu1[a] = 0.0 + /* "trunk/gensim/models/word2vec_inner.pyx":204 + * + * memset(neu1, 0, size * cython.sizeof(REAL_t)) * count = 0.0 # <<<<<<<<<<<<<< * for m in range(j, k): * if m == i or codelens[m] == 0: */ __pyx_v_count = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - /* "trunk/gensim/models/word2vec_inner.pyx":533 - * neu1[a] = 0.0 + /* "trunk/gensim/models/word2vec_inner.pyx":205 + * memset(neu1, 0, size * cython.sizeof(REAL_t)) * count = 0.0 * for m in range(j, k): # <<<<<<<<<<<<<< * if m == i or codelens[m] == 0: * continue */ __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; + for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { + __pyx_v_m = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":534 + /* "trunk/gensim/models/word2vec_inner.pyx":206 * count = 0.0 * for m in range(j, k): * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< * continue * else: */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { + __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); + if (!__pyx_t_4) { } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L8_bool_binop_done; + __pyx_t_3 = __pyx_t_4; + goto __pyx_L6_bool_binop_done; } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L8_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L6_bool_binop_done:; + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":535 + /* "trunk/gensim/models/word2vec_inner.pyx":207 * for m in range(j, k): * if m == i or codelens[m] == 0: * continue # <<<<<<<<<<<<<< * else: * count += ONEF */ - goto __pyx_L5_continue; + goto __pyx_L3_continue; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":537 + /* "trunk/gensim/models/word2vec_inner.pyx":209 * continue * else: * count += ONEF # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): */ __pyx_v_count = (__pyx_v_count + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF); - /* "trunk/gensim/models/word2vec_inner.pyx":538 + /* "trunk/gensim/models/word2vec_inner.pyx":210 * else: * count += ONEF - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) # <<<<<<<<<<<<<< + * if count > (0.5): + * inv_count = ONEF/count */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + } + __pyx_L3_continue:; + } - /* "trunk/gensim/models/word2vec_inner.pyx":539 + /* "trunk/gensim/models/word2vec_inner.pyx":211 * count += ONEF - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] # <<<<<<<<<<<<<< - * if cbow_mean and count > (0.5): - * for a in range(size): + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): # <<<<<<<<<<<<<< + * inv_count = ONEF/count + * if cbow_mean: */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) + (__pyx_v_syn0[(((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a)])); - } - } - __pyx_L5_continue:; - } + __pyx_t_3 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":540 - * for a in range(size): - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): # <<<<<<<<<<<<<< - * for a in range(size): - * neu1[a] /= count + /* "trunk/gensim/models/word2vec_inner.pyx":212 + * our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + * if count > (0.5): + * inv_count = ONEF/count # <<<<<<<<<<<<<< + * if cbow_mean: + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) */ - __pyx_t_5 = (__pyx_v_cbow_mean != 0); - if (__pyx_t_5) { - } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L13_bool_binop_done; + __pyx_v_inv_count = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF / __pyx_v_count); + goto __pyx_L8; } - __pyx_t_5 = ((__pyx_v_count > ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.5)) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L13_bool_binop_done:; - if (__pyx_t_4) { + __pyx_L8:; - /* "trunk/gensim/models/word2vec_inner.pyx":541 - * neu1[a] += syn0[indexes[m] * size + a] - * if cbow_mean and count > (0.5): - * for a in range(size): # <<<<<<<<<<<<<< - * neu1[a] /= count + /* "trunk/gensim/models/word2vec_inner.pyx":213 + * if count > (0.5): + * inv_count = ONEF/count + * if cbow_mean: # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) * */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_3 = (__pyx_v_cbow_mean != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":542 - * if cbow_mean and count > (0.5): - * for a in range(size): - * neu1[a] /= count # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":214 + * inv_count = ONEF/count + * if cbow_mean: + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< * - * for a in range(size): + * memset(work, 0, size * cython.sizeof(REAL_t)) */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_neu1[__pyx_t_7]) = ((__pyx_v_neu1[__pyx_t_7]) / __pyx_v_count); - } - goto __pyx_L12; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + goto __pyx_L9; } - __pyx_L12:; - - /* "trunk/gensim/models/word2vec_inner.pyx":544 - * neu1[a] /= count - * - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] = 0.0 - * - */ - __pyx_t_1 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_L9:; - /* "trunk/gensim/models/word2vec_inner.pyx":545 + /* "trunk/gensim/models/word2vec_inner.pyx":216 + * sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) * - * for a in range(size): - * work[a] = 0.0 # <<<<<<<<<<<<<< + * memset(work, 0, size * cython.sizeof(REAL_t)) # <<<<<<<<<<<<<< * * for d in range(negative+1): */ - (__pyx_v_work[__pyx_v_a]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - } + memset(__pyx_v_work, 0, (__pyx_v_size * (sizeof(__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)))); - /* "trunk/gensim/models/word2vec_inner.pyx":547 - * work[a] = 0.0 + /* "trunk/gensim/models/word2vec_inner.pyx":218 + * memset(work, 0, size * cython.sizeof(REAL_t)) * * for d in range(negative+1): # <<<<<<<<<<<<<< * if d == 0: * target_index = word_index */ - __pyx_t_8 = (__pyx_v_negative + 1); - for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_8; __pyx_t_1+=1) { + __pyx_t_5 = (__pyx_v_negative + 1); + for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_5; __pyx_t_1+=1) { __pyx_v_d = __pyx_t_1; - /* "trunk/gensim/models/word2vec_inner.pyx":548 + /* "trunk/gensim/models/word2vec_inner.pyx":219 * * for d in range(negative+1): * if d == 0: # <<<<<<<<<<<<<< * target_index = word_index * label = ONEF */ - __pyx_t_4 = ((__pyx_v_d == 0) != 0); - if (__pyx_t_4) { + __pyx_t_3 = ((__pyx_v_d == 0) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":549 + /* "trunk/gensim/models/word2vec_inner.pyx":220 * for d in range(negative+1): * if d == 0: * target_index = word_index # <<<<<<<<<<<<<< @@ -4869,7 +2508,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_target_index = __pyx_v_word_index; - /* "trunk/gensim/models/word2vec_inner.pyx":550 + /* "trunk/gensim/models/word2vec_inner.pyx":221 * if d == 0: * target_index = word_index * label = ONEF # <<<<<<<<<<<<<< @@ -4877,11 +2516,11 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas * target_index = table[(next_random >> 16) % table_len] */ __pyx_v_label = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF; - goto __pyx_L21; + goto __pyx_L12; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":552 + /* "trunk/gensim/models/word2vec_inner.pyx":223 * label = ONEF * else: * target_index = table[(next_random >> 16) % table_len] # <<<<<<<<<<<<<< @@ -4890,7 +2529,7 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_target_index = (__pyx_v_table[((__pyx_v_next_random >> 16) % __pyx_v_table_len)]); - /* "trunk/gensim/models/word2vec_inner.pyx":553 + /* "trunk/gensim/models/word2vec_inner.pyx":224 * else: * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo # <<<<<<<<<<<<<< @@ -4899,27 +2538,27 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_next_random = (((__pyx_v_next_random * ((unsigned PY_LONG_LONG)25214903917ULL)) + 11) & __pyx_v_modulo); - /* "trunk/gensim/models/word2vec_inner.pyx":554 + /* "trunk/gensim/models/word2vec_inner.pyx":225 * target_index = table[(next_random >> 16) % table_len] * next_random = (next_random * 25214903917ULL + 11) & modulo * if target_index == word_index: # <<<<<<<<<<<<<< * continue * label = 0.0 */ - __pyx_t_4 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); - if (__pyx_t_4) { + __pyx_t_3 = ((__pyx_v_target_index == __pyx_v_word_index) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":555 + /* "trunk/gensim/models/word2vec_inner.pyx":226 * next_random = (next_random * 25214903917ULL + 11) & modulo * if target_index == word_index: * continue # <<<<<<<<<<<<<< * label = 0.0 * */ - goto __pyx_L19_continue; + goto __pyx_L10_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":556 + /* "trunk/gensim/models/word2vec_inner.pyx":227 * if target_index == word_index: * continue * label = 0.0 # <<<<<<<<<<<<<< @@ -4928,218 +2567,180 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas */ __pyx_v_label = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); } - __pyx_L21:; + __pyx_L12:; - /* "trunk/gensim/models/word2vec_inner.pyx":558 + /* "trunk/gensim/models/word2vec_inner.pyx":229 * label = 0.0 * * row2 = target_index * size # <<<<<<<<<<<<<< - * f = 0.0 - * for a in range(size): + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + * if f <= -MAX_EXP or f >= MAX_EXP: */ __pyx_v_row2 = (__pyx_v_target_index * __pyx_v_size); - /* "trunk/gensim/models/word2vec_inner.pyx":559 + /* "trunk/gensim/models/word2vec_inner.pyx":230 * * row2 = target_index * size - * f = 0.0 # <<<<<<<<<<<<<< - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] - */ - __pyx_v_f = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)0.0); - - /* "trunk/gensim/models/word2vec_inner.pyx":560 - * row2 = target_index * size - * f = 0.0 - * for a in range(size): # <<<<<<<<<<<<<< - * f += neu1[a] * syn1neg[row2 + a] - * if f <= -MAX_EXP or f >= MAX_EXP: - */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":561 - * f = 0.0 - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] # <<<<<<<<<<<<<< + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< * if f <= -MAX_EXP or f >= MAX_EXP: * continue */ - __pyx_v_f = (__pyx_v_f + ((__pyx_v_neu1[__pyx_v_a]) * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } + __pyx_v_f = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot((&__pyx_v_size), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":562 - * for a in range(size): - * f += neu1[a] * syn1neg[row2 + a] + /* "trunk/gensim/models/word2vec_inner.pyx":231 + * row2 = target_index * size + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: # <<<<<<<<<<<<<< * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] */ - __pyx_t_5 = ((__pyx_v_f <= -6.0) != 0); - if (!__pyx_t_5) { + __pyx_t_4 = ((__pyx_v_f <= -6.0) != 0); + if (!__pyx_t_4) { } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L26_bool_binop_done; + __pyx_t_3 = __pyx_t_4; + goto __pyx_L15_bool_binop_done; } - __pyx_t_5 = ((__pyx_v_f >= 6.0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L26_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = ((__pyx_v_f >= 6.0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L15_bool_binop_done:; + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":563 - * f += neu1[a] * syn1neg[row2 + a] + /* "trunk/gensim/models/word2vec_inner.pyx":232 + * f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) * if f <= -MAX_EXP or f >= MAX_EXP: * continue # <<<<<<<<<<<<<< * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha */ - goto __pyx_L19_continue; + goto __pyx_L10_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":564 + /* "trunk/gensim/models/word2vec_inner.pyx":233 * if f <= -MAX_EXP or f >= MAX_EXP: * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] # <<<<<<<<<<<<<< * g = (label - f) * alpha - * for a in range(size): + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) */ __pyx_v_f = (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[((int)((__pyx_v_f + 6.0) * 83.0))]); - /* "trunk/gensim/models/word2vec_inner.pyx":565 + /* "trunk/gensim/models/word2vec_inner.pyx":234 * continue * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha # <<<<<<<<<<<<<< - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) */ __pyx_v_g = ((__pyx_v_label - __pyx_v_f) * __pyx_v_alpha); - /* "trunk/gensim/models/word2vec_inner.pyx":566 + /* "trunk/gensim/models/word2vec_inner.pyx":235 * f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] * g = (label - f) * alpha - * for a in range(size): # <<<<<<<<<<<<<< - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) # <<<<<<<<<<<<<< + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":567 + /* "trunk/gensim/models/word2vec_inner.pyx":236 * g = (label - f) * alpha - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] # <<<<<<<<<<<<<< - * for a in range(size): - * syn1neg[row2 + a] += g * neu1[a] + * our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) # <<<<<<<<<<<<<< + * + * if not cbow_mean: # divide error over summed window vectors */ - __pyx_t_7 = __pyx_v_a; - (__pyx_v_work[__pyx_t_7]) = ((__pyx_v_work[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_syn1neg[(__pyx_v_row2 + __pyx_v_a)]))); - } + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&__pyx_v_g), __pyx_v_neu1, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn1neg[__pyx_v_row2])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + __pyx_L10_continue:; + } - /* "trunk/gensim/models/word2vec_inner.pyx":568 - * for a in range(size): - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): # <<<<<<<<<<<<<< - * syn1neg[row2 + a] += g * neu1[a] + /* "trunk/gensim/models/word2vec_inner.pyx":238 + * our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + * + * if not cbow_mean: # divide error over summed window vectors # <<<<<<<<<<<<<< + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) * */ - __pyx_t_3 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_3; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; + __pyx_t_3 = ((!(__pyx_v_cbow_mean != 0)) != 0); + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":569 - * work[a] += g * syn1neg[row2 + a] - * for a in range(size): - * syn1neg[row2 + a] += g * neu1[a] # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":239 * - * for m in range(j, k): + * if not cbow_mean: # divide error over summed window vectors + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) # <<<<<<<<<<<<<< + * + * for m in range(j,k): */ - __pyx_t_7 = (__pyx_v_row2 + __pyx_v_a); - (__pyx_v_syn1neg[__pyx_t_7]) = ((__pyx_v_syn1neg[__pyx_t_7]) + (__pyx_v_g * (__pyx_v_neu1[__pyx_v_a]))); - } - __pyx_L19_continue:; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal((&__pyx_v_size), (&__pyx_v_inv_count), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); + goto __pyx_L17; } + __pyx_L17:; - /* "trunk/gensim/models/word2vec_inner.pyx":571 - * syn1neg[row2 + a] += g * neu1[a] + /* "trunk/gensim/models/word2vec_inner.pyx":241 + * sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) * - * for m in range(j, k): # <<<<<<<<<<<<<< + * for m in range(j,k): # <<<<<<<<<<<<<< * if m == i or codelens[m] == 0: * continue */ __pyx_t_1 = __pyx_v_k; - for (__pyx_t_3 = __pyx_v_j; __pyx_t_3 < __pyx_t_1; __pyx_t_3+=1) { - __pyx_v_m = __pyx_t_3; + for (__pyx_t_2 = __pyx_v_j; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { + __pyx_v_m = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":572 + /* "trunk/gensim/models/word2vec_inner.pyx":242 * - * for m in range(j, k): + * for m in range(j,k): * if m == i or codelens[m] == 0: # <<<<<<<<<<<<<< * continue * else: */ - __pyx_t_5 = ((__pyx_v_m == __pyx_v_i) != 0); - if (!__pyx_t_5) { + __pyx_t_4 = ((__pyx_v_m == __pyx_v_i) != 0); + if (!__pyx_t_4) { } else { - __pyx_t_4 = __pyx_t_5; - goto __pyx_L35_bool_binop_done; + __pyx_t_3 = __pyx_t_4; + goto __pyx_L21_bool_binop_done; } - __pyx_t_5 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); - __pyx_t_4 = __pyx_t_5; - __pyx_L35_bool_binop_done:; - if (__pyx_t_4) { + __pyx_t_4 = (((__pyx_v_codelens[__pyx_v_m]) == 0) != 0); + __pyx_t_3 = __pyx_t_4; + __pyx_L21_bool_binop_done:; + if (__pyx_t_3) { - /* "trunk/gensim/models/word2vec_inner.pyx":573 - * for m in range(j, k): + /* "trunk/gensim/models/word2vec_inner.pyx":243 + * for m in range(j,k): * if m == i or codelens[m] == 0: * continue # <<<<<<<<<<<<<< * else: - * for a in range(size): + * our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m]*size], &ONE) */ - goto __pyx_L32_continue; + goto __pyx_L18_continue; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":575 + /* "trunk/gensim/models/word2vec_inner.pyx":245 * continue * else: - * for a in range(size): # <<<<<<<<<<<<<< - * syn0[indexes[m] * size + a] += work[a] - * - */ - __pyx_t_6 = __pyx_v_size; - for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_6; __pyx_t_2+=1) { - __pyx_v_a = __pyx_t_2; - - /* "trunk/gensim/models/word2vec_inner.pyx":576 - * else: - * for a in range(size): - * syn0[indexes[m] * size + a] += work[a] # <<<<<<<<<<<<<< + * our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m]*size], &ONE) # <<<<<<<<<<<<<< * * return next_random */ - __pyx_t_7 = (((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size) + __pyx_v_a); - (__pyx_v_syn0[__pyx_t_7]) = ((__pyx_v_syn0[__pyx_t_7]) + (__pyx_v_work[__pyx_v_a])); - } + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy((&__pyx_v_size), (&(__pyx_v_word_locks[(__pyx_v_indexes[__pyx_v_m])])), __pyx_v_work, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), (&(__pyx_v_syn0[((__pyx_v_indexes[__pyx_v_m]) * __pyx_v_size)])), (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); } - __pyx_L32_continue:; + __pyx_L18_continue:; } - /* "trunk/gensim/models/word2vec_inner.pyx":578 - * syn0[indexes[m] * size + a] += work[a] + /* "trunk/gensim/models/word2vec_inner.pyx":247 + * our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m]*size], &ONE) * * return next_random # <<<<<<<<<<<<<< * - * def train_sentence_sg(model, sentence, alpha, _work): + * */ __pyx_r = __pyx_v_next_random; goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":515 - * return next_random + /* "trunk/gensim/models/word2vec_inner.pyx":188 + * * - * cdef unsigned long long fast_sentence2_cbow_neg( # <<<<<<<<<<<<<< + * cdef unsigned long long fast_sentence_cbow_neg( # <<<<<<<<<<<<<< * const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], * REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, */ @@ -5149,8 +2750,8 @@ static unsigned PY_LONG_LONG __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fas return __pyx_r; } -/* "trunk/gensim/models/word2vec_inner.pyx":580 - * return next_random +/* "trunk/gensim/models/word2vec_inner.pyx":250 + * * * def train_sentence_sg(model, sentence, alpha, _work): # <<<<<<<<<<<<<< * cdef int hs = model.hs @@ -5193,21 +2794,21 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_14word2vec_inner_1train_sentenc case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_alpha)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_sg") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_sg") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 4) { goto __pyx_L5_argtuple_error; @@ -5224,7 +2825,7 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_14word2vec_inner_1train_sentenc } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_sg", 1, 4, 4, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("trunk.gensim.models.word2vec_inner.train_sentence_sg", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5241,6 +2842,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence int __pyx_v_hs; int __pyx_v_negative; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v__alpha; int __pyx_v_size; @@ -5288,82 +2890,95 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence int __pyx_clineno = 0; __Pyx_RefNannySetupContext("train_sentence_sg", 0); - /* "trunk/gensim/models/word2vec_inner.pyx":581 + /* "trunk/gensim/models/word2vec_inner.pyx":251 * * def train_sentence_sg(model, sentence, alpha, _work): * cdef int hs = model.hs # <<<<<<<<<<<<<< * cdef int negative = model.negative * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 251; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 581; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 251; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_hs = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":582 + /* "trunk/gensim/models/word2vec_inner.pyx":252 * def train_sentence_sg(model, sentence, alpha, _work): * cdef int hs = model.hs * cdef int negative = model.negative # <<<<<<<<<<<<<< * * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 582; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 252; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 582; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 252; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_negative = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":584 + /* "trunk/gensim/models/word2vec_inner.pyx":254 * cdef int negative = model.negative * * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) # <<<<<<<<<<<<<< + * cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) * cdef REAL_t *work - * cdef REAL_t _alpha = alpha */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 584; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 254; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 584; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 254; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn0 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":586 + /* "trunk/gensim/models/word2vec_inner.pyx":255 + * * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) + * cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) # <<<<<<<<<<<<<< + * cdef REAL_t *work + * cdef REAL_t _alpha = alpha + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0_lockf); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_word_locks = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "trunk/gensim/models/word2vec_inner.pyx":257 + * cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) * cdef REAL_t *work * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< * cdef int size = model.layer1_size * */ - __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 586; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 257; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v__alpha = __pyx_t_3; - /* "trunk/gensim/models/word2vec_inner.pyx":587 + /* "trunk/gensim/models/word2vec_inner.pyx":258 * cdef REAL_t *work * cdef REAL_t _alpha = alpha * cdef int size = model.layer1_size # <<<<<<<<<<<<<< * * cdef int codelens[MAX_SENTENCE_LEN] */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 587; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 587; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_size = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":593 + /* "trunk/gensim/models/word2vec_inner.pyx":264 * cdef np.uint32_t reduced_windows[MAX_SENTENCE_LEN] * cdef int sentence_len * cdef int window = model.window # <<<<<<<<<<<<<< * * cdef int i, j, k */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 593; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 264; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 593; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 264; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_window = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":596 + /* "trunk/gensim/models/word2vec_inner.pyx":267 * * cdef int i, j, k * cdef long result = 0 # <<<<<<<<<<<<<< @@ -5372,7 +2987,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence */ __pyx_v_result = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":609 + /* "trunk/gensim/models/word2vec_inner.pyx":280 * cdef unsigned long long next_random * * if hs: # <<<<<<<<<<<<<< @@ -5382,23 +2997,23 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_4 = (__pyx_v_hs != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":610 + /* "trunk/gensim/models/word2vec_inner.pyx":281 * * if hs: * syn1 = (np.PyArray_DATA(model.syn1)) # <<<<<<<<<<<<<< * * if negative: */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 610; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 281; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 610; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 281; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; goto __pyx_L3; } __pyx_L3:; - /* "trunk/gensim/models/word2vec_inner.pyx":612 + /* "trunk/gensim/models/word2vec_inner.pyx":283 * syn1 = (np.PyArray_DATA(model.syn1)) * * if negative: # <<<<<<<<<<<<<< @@ -5408,106 +3023,106 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_4 = (__pyx_v_negative != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":613 + /* "trunk/gensim/models/word2vec_inner.pyx":284 * * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) # <<<<<<<<<<<<<< * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 613; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 284; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 613; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 284; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1neg = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":614 + /* "trunk/gensim/models/word2vec_inner.pyx":285 * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) # <<<<<<<<<<<<<< * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 614; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 285; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 614; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 285; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_table = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":615 + /* "trunk/gensim/models/word2vec_inner.pyx":286 * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) # <<<<<<<<<<<<<< * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 615; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 286; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 615; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 286; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_table_len = __pyx_t_5; - /* "trunk/gensim/models/word2vec_inner.pyx":616 + /* "trunk/gensim/models/word2vec_inner.pyx":287 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_v_next_random = __pyx_t_8; goto __pyx_L4; } __pyx_L4:; - /* "trunk/gensim/models/word2vec_inner.pyx":619 + /* "trunk/gensim/models/word2vec_inner.pyx":290 * * # convert Python structures to primitive types, so we can release the GIL * work = np.PyArray_DATA(_work) # <<<<<<<<<<<<<< * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) * */ - if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 619; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 290; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_work = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__work))); - /* "trunk/gensim/models/word2vec_inner.pyx":620 + /* "trunk/gensim/models/word2vec_inner.pyx":291 * # convert Python structures to primitive types, so we can release the GIL * work = np.PyArray_DATA(_work) * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) # <<<<<<<<<<<<<< * * for i in range(sentence_len): */ - __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 620; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 291; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_9 = 10000; if (((__pyx_t_5 < __pyx_t_9) != 0)) { __pyx_t_10 = __pyx_t_5; @@ -5516,7 +3131,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } __pyx_v_sentence_len = ((int)__pyx_t_10); - /* "trunk/gensim/models/word2vec_inner.pyx":622 + /* "trunk/gensim/models/word2vec_inner.pyx":293 * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) * * for i in range(sentence_len): # <<<<<<<<<<<<<< @@ -5527,19 +3142,19 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { __pyx_v_i = __pyx_t_11; - /* "trunk/gensim/models/word2vec_inner.pyx":623 + /* "trunk/gensim/models/word2vec_inner.pyx":294 * * for i in range(sentence_len): * word = sentence[i] # <<<<<<<<<<<<<< * if word is None: * codelens[i] = 0 */ - __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 623; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 294; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __Pyx_GOTREF(__pyx_t_6); __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":624 + /* "trunk/gensim/models/word2vec_inner.pyx":295 * for i in range(sentence_len): * word = sentence[i] * if word is None: # <<<<<<<<<<<<<< @@ -5550,7 +3165,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_12 = (__pyx_t_4 != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":625 + /* "trunk/gensim/models/word2vec_inner.pyx":296 * word = sentence[i] * if word is None: * codelens[i] = 0 # <<<<<<<<<<<<<< @@ -5562,20 +3177,20 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":627 + /* "trunk/gensim/models/word2vec_inner.pyx":298 * codelens[i] = 0 * else: * indexes[i] = word.index # <<<<<<<<<<<<<< * if hs: * codelens[i] = len(word.code) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 627; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 298; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 627; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 298; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; (__pyx_v_indexes[__pyx_v_i]) = __pyx_t_13; - /* "trunk/gensim/models/word2vec_inner.pyx":628 + /* "trunk/gensim/models/word2vec_inner.pyx":299 * else: * indexes[i] = word.index * if hs: # <<<<<<<<<<<<<< @@ -5585,49 +3200,49 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_12 = (__pyx_v_hs != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":629 + /* "trunk/gensim/models/word2vec_inner.pyx":300 * indexes[i] = word.index * if hs: * codelens[i] = len(word.code) # <<<<<<<<<<<<<< * codes[i] = np.PyArray_DATA(word.code) * points[i] = np.PyArray_DATA(word.point) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 629; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 300; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_10 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 629; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_10 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 300; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; (__pyx_v_codelens[__pyx_v_i]) = ((int)__pyx_t_10); - /* "trunk/gensim/models/word2vec_inner.pyx":630 + /* "trunk/gensim/models/word2vec_inner.pyx":301 * if hs: * codelens[i] = len(word.code) * codes[i] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< * points[i] = np.PyArray_DATA(word.point) * else: */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 630; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 301; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 630; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 301; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_codes[__pyx_v_i]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":631 + /* "trunk/gensim/models/word2vec_inner.pyx":302 * codelens[i] = len(word.code) * codes[i] = np.PyArray_DATA(word.code) * points[i] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< * else: * codelens[i] = 1 */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 631; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 302; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 631; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 302; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_points[__pyx_v_i]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; goto __pyx_L8; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":633 + /* "trunk/gensim/models/word2vec_inner.pyx":304 * points[i] = np.PyArray_DATA(word.point) * else: * codelens[i] = 1 # <<<<<<<<<<<<<< @@ -5638,7 +3253,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } __pyx_L8:; - /* "trunk/gensim/models/word2vec_inner.pyx":634 + /* "trunk/gensim/models/word2vec_inner.pyx":305 * else: * codelens[i] = 1 * result += 1 # <<<<<<<<<<<<<< @@ -5650,7 +3265,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_L7:; } - /* "trunk/gensim/models/word2vec_inner.pyx":636 + /* "trunk/gensim/models/word2vec_inner.pyx":307 * result += 1 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< @@ -5658,17 +3273,17 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence * */ __pyx_t_2 = 0; - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_14); __pyx_t_15 = NULL; __pyx_t_10 = 0; @@ -5682,7 +3297,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_10 = 1; } } - __pyx_t_16 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_16 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_15) { PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_15); __Pyx_GIVEREF(__pyx_t_15); __pyx_t_15 = NULL; @@ -5696,7 +3311,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __Pyx_GIVEREF(__pyx_t_14); __pyx_t_1 = 0; __pyx_t_14 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -5704,9 +3319,9 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); __pyx_t_10 = 0; __pyx_t_17 = NULL; } else { - __pyx_t_10 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_10 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; for (;;) { @@ -5714,16 +3329,16 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } else { if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } } else { @@ -5732,7 +3347,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } break; } @@ -5743,17 +3358,17 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_v_i = __pyx_t_2; __pyx_t_2 = (__pyx_t_2 + 1); - /* "trunk/gensim/models/word2vec_inner.pyx":637 + /* "trunk/gensim/models/word2vec_inner.pyx":308 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): * reduced_windows[i] = item # <<<<<<<<<<<<<< * * # release GIL & train on the sentence */ - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 637; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 308; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_13; - /* "trunk/gensim/models/word2vec_inner.pyx":636 + /* "trunk/gensim/models/word2vec_inner.pyx":307 * result += 1 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< @@ -5763,7 +3378,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":640 + /* "trunk/gensim/models/word2vec_inner.pyx":311 * * # release GIL & train on the sentence * with nogil: # <<<<<<<<<<<<<< @@ -5777,7 +3392,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence #endif /*try:*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":641 + /* "trunk/gensim/models/word2vec_inner.pyx":312 * # release GIL & train on the sentence * with nogil: * for i in range(sentence_len): # <<<<<<<<<<<<<< @@ -5788,7 +3403,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { __pyx_v_i = __pyx_t_11; - /* "trunk/gensim/models/word2vec_inner.pyx":642 + /* "trunk/gensim/models/word2vec_inner.pyx":313 * with nogil: * for i in range(sentence_len): * if codelens[i] == 0: # <<<<<<<<<<<<<< @@ -5798,7 +3413,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_12 = (((__pyx_v_codelens[__pyx_v_i]) == 0) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":643 + /* "trunk/gensim/models/word2vec_inner.pyx":314 * for i in range(sentence_len): * if codelens[i] == 0: * continue # <<<<<<<<<<<<<< @@ -5808,7 +3423,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence goto __pyx_L14_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":644 + /* "trunk/gensim/models/word2vec_inner.pyx":315 * if codelens[i] == 0: * continue * j = i - window + reduced_windows[i] # <<<<<<<<<<<<<< @@ -5817,7 +3432,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence */ __pyx_v_j = ((__pyx_v_i - __pyx_v_window) + (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/word2vec_inner.pyx":645 + /* "trunk/gensim/models/word2vec_inner.pyx":316 * continue * j = i - window + reduced_windows[i] * if j < 0: # <<<<<<<<<<<<<< @@ -5827,7 +3442,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_12 = ((__pyx_v_j < 0) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":646 + /* "trunk/gensim/models/word2vec_inner.pyx":317 * j = i - window + reduced_windows[i] * if j < 0: * j = 0 # <<<<<<<<<<<<<< @@ -5839,7 +3454,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } __pyx_L17:; - /* "trunk/gensim/models/word2vec_inner.pyx":647 + /* "trunk/gensim/models/word2vec_inner.pyx":318 * if j < 0: * j = 0 * k = i + window + 1 - reduced_windows[i] # <<<<<<<<<<<<<< @@ -5848,7 +3463,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence */ __pyx_v_k = (((__pyx_v_i + __pyx_v_window) + 1) - (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/word2vec_inner.pyx":648 + /* "trunk/gensim/models/word2vec_inner.pyx":319 * j = 0 * k = i + window + 1 - reduced_windows[i] * if k > sentence_len: # <<<<<<<<<<<<<< @@ -5858,7 +3473,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_t_12 = ((__pyx_v_k > __pyx_v_sentence_len) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":649 + /* "trunk/gensim/models/word2vec_inner.pyx":320 * k = i + window + 1 - reduced_windows[i] * if k > sentence_len: * k = sentence_len # <<<<<<<<<<<<<< @@ -5870,7 +3485,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } __pyx_L18:; - /* "trunk/gensim/models/word2vec_inner.pyx":650 + /* "trunk/gensim/models/word2vec_inner.pyx":321 * if k > sentence_len: * k = sentence_len * for j in range(j, k): # <<<<<<<<<<<<<< @@ -5881,7 +3496,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence for (__pyx_t_19 = __pyx_v_j; __pyx_t_19 < __pyx_t_18; __pyx_t_19+=1) { __pyx_v_j = __pyx_t_19; - /* "trunk/gensim/models/word2vec_inner.pyx":651 + /* "trunk/gensim/models/word2vec_inner.pyx":322 * k = sentence_len * for j in range(j, k): * if j == i or codelens[j] == 0: # <<<<<<<<<<<<<< @@ -5899,56 +3514,56 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence __pyx_L22_bool_binop_done:; if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":652 + /* "trunk/gensim/models/word2vec_inner.pyx":323 * for j in range(j, k): * if j == i or codelens[j] == 0: * continue # <<<<<<<<<<<<<< * if hs: - * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) + * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) */ goto __pyx_L19_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":653 + /* "trunk/gensim/models/word2vec_inner.pyx":324 * if j == i or codelens[j] == 0: * continue * if hs: # <<<<<<<<<<<<<< - * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) + * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) * if negative: */ __pyx_t_12 = (__pyx_v_hs != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":654 + /* "trunk/gensim/models/word2vec_inner.pyx":325 * continue * if hs: - * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) # <<<<<<<<<<<<<< + * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) # <<<<<<<<<<<<<< * if negative: - * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random) + * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random, word_locks) */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work); + __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), (__pyx_v_codelens[__pyx_v_i]), __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work, __pyx_v_word_locks); goto __pyx_L24; } __pyx_L24:; - /* "trunk/gensim/models/word2vec_inner.pyx":655 + /* "trunk/gensim/models/word2vec_inner.pyx":326 * if hs: - * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) + * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) * if negative: # <<<<<<<<<<<<<< - * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random) + * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random, word_locks) * */ __pyx_t_12 = (__pyx_v_negative != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":656 - * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) + /* "trunk/gensim/models/word2vec_inner.pyx":327 + * fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) * if negative: - * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random) # <<<<<<<<<<<<<< + * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random, word_locks) # <<<<<<<<<<<<<< * * return result */ - __pyx_v_next_random = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, (__pyx_v_indexes[__pyx_v_i]), (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work, __pyx_v_next_random); + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, (__pyx_v_indexes[__pyx_v_i]), (__pyx_v_indexes[__pyx_v_j]), __pyx_v__alpha, __pyx_v_work, __pyx_v_next_random, __pyx_v_word_locks); goto __pyx_L25; } __pyx_L25:; @@ -5958,7 +3573,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } } - /* "trunk/gensim/models/word2vec_inner.pyx":640 + /* "trunk/gensim/models/word2vec_inner.pyx":311 * * # release GIL & train on the sentence * with nogil: # <<<<<<<<<<<<<< @@ -5976,22 +3591,22 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence } } - /* "trunk/gensim/models/word2vec_inner.pyx":658 - * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random) + /* "trunk/gensim/models/word2vec_inner.pyx":329 + * next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random, word_locks) * * return result # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 658; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __pyx_r = __pyx_t_7; __pyx_t_7 = 0; goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":580 - * return next_random + /* "trunk/gensim/models/word2vec_inner.pyx":250 + * * * def train_sentence_sg(model, sentence, alpha, _work): # <<<<<<<<<<<<<< * cdef int hs = model.hs @@ -6016,7 +3631,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_train_sentence return __pyx_r; } -/* "trunk/gensim/models/word2vec_inner.pyx":661 +/* "trunk/gensim/models/word2vec_inner.pyx":332 * * * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): # <<<<<<<<<<<<<< @@ -6062,26 +3677,26 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_14word2vec_inner_3train_sentenc case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_alpha)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_work)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 4: if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_neu1)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_cbow") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "train_sentence_cbow") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 5) { goto __pyx_L5_argtuple_error; @@ -6100,7 +3715,7 @@ static PyObject *__pyx_pw_5trunk_6gensim_6models_14word2vec_inner_3train_sentenc } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("train_sentence_cbow", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("trunk.gensim.models.word2vec_inner.train_sentence_cbow", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -6118,6 +3733,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc int __pyx_v_negative; int __pyx_v_cbow_mean; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_syn0; + __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_word_locks; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_work; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *__pyx_v_neu1; __pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t __pyx_v__alpha; @@ -6164,95 +3780,108 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc int __pyx_clineno = 0; __Pyx_RefNannySetupContext("train_sentence_cbow", 0); - /* "trunk/gensim/models/word2vec_inner.pyx":662 + /* "trunk/gensim/models/word2vec_inner.pyx":333 * * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): * cdef int hs = model.hs # <<<<<<<<<<<<<< * cdef int negative = model.negative * cdef int cbow_mean = model.cbow_mean */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 662; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_hs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 662; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 333; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_hs = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":663 + /* "trunk/gensim/models/word2vec_inner.pyx":334 * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): * cdef int hs = model.hs * cdef int negative = model.negative # <<<<<<<<<<<<<< * cdef int cbow_mean = model.cbow_mean * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 663; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_negative); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 334; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 663; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 334; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_negative = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":664 + /* "trunk/gensim/models/word2vec_inner.pyx":335 * cdef int hs = model.hs * cdef int negative = model.negative * cdef int cbow_mean = model.cbow_mean # <<<<<<<<<<<<<< * * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_cbow_mean); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 664; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_cbow_mean); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 335; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 664; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 335; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_cbow_mean = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":666 + /* "trunk/gensim/models/word2vec_inner.pyx":337 * cdef int cbow_mean = model.cbow_mean * * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) # <<<<<<<<<<<<<< + * cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) * cdef REAL_t *work - * cdef REAL_t *neu1 */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 666; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 337; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 666; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 337; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn0 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":669 + /* "trunk/gensim/models/word2vec_inner.pyx":338 + * + * cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) + * cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) # <<<<<<<<<<<<<< + * cdef REAL_t *work + * cdef REAL_t *neu1 + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn0_lockf); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 338; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 338; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_word_locks = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "trunk/gensim/models/word2vec_inner.pyx":341 * cdef REAL_t *work * cdef REAL_t *neu1 * cdef REAL_t _alpha = alpha # <<<<<<<<<<<<<< * cdef int size = model.layer1_size * */ - __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 669; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_alpha); if (unlikely((__pyx_t_3 == (npy_float32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 341; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v__alpha = __pyx_t_3; - /* "trunk/gensim/models/word2vec_inner.pyx":670 + /* "trunk/gensim/models/word2vec_inner.pyx":342 * cdef REAL_t *neu1 * cdef REAL_t _alpha = alpha * cdef int size = model.layer1_size # <<<<<<<<<<<<<< * * cdef int codelens[MAX_SENTENCE_LEN] */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 670; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_layer1_size); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 342; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 670; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 342; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_size = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":676 + /* "trunk/gensim/models/word2vec_inner.pyx":348 * cdef np.uint32_t reduced_windows[MAX_SENTENCE_LEN] * cdef int sentence_len * cdef int window = model.window # <<<<<<<<<<<<<< * * cdef int i, j, k */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 676; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 348; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 676; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 348; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_window = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":679 + /* "trunk/gensim/models/word2vec_inner.pyx":351 * * cdef int i, j, k * cdef long result = 0 # <<<<<<<<<<<<<< @@ -6261,7 +3890,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc */ __pyx_v_result = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":692 + /* "trunk/gensim/models/word2vec_inner.pyx":364 * cdef unsigned long long next_random * * if hs: # <<<<<<<<<<<<<< @@ -6271,23 +3900,23 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_4 = (__pyx_v_hs != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":693 + /* "trunk/gensim/models/word2vec_inner.pyx":365 * * if hs: * syn1 = (np.PyArray_DATA(model.syn1)) # <<<<<<<<<<<<<< * * if negative: */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 693; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 365; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 693; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 365; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; goto __pyx_L3; } __pyx_L3:; - /* "trunk/gensim/models/word2vec_inner.pyx":695 + /* "trunk/gensim/models/word2vec_inner.pyx":367 * syn1 = (np.PyArray_DATA(model.syn1)) * * if negative: # <<<<<<<<<<<<<< @@ -6297,116 +3926,116 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_4 = (__pyx_v_negative != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":696 + /* "trunk/gensim/models/word2vec_inner.pyx":368 * * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) # <<<<<<<<<<<<<< * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 696; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_syn1neg); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 368; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 696; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 368; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_syn1neg = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":697 + /* "trunk/gensim/models/word2vec_inner.pyx":369 * if negative: * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) # <<<<<<<<<<<<<< * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 697; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 369; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 697; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 369; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_table = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_1))); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":698 + /* "trunk/gensim/models/word2vec_inner.pyx":370 * syn1neg = (np.PyArray_DATA(model.syn1neg)) * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) # <<<<<<<<<<<<<< * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 698; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_n_s_table); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 370; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 698; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 370; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_table_len = __pyx_t_5; - /* "trunk/gensim/models/word2vec_inner.pyx":699 + /* "trunk/gensim/models/word2vec_inner.pyx":371 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_random); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_randint); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Multiply(__pyx_int_16777216, __pyx_t_6); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_random); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_randint); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyNumber_Add(__pyx_t_1, __pyx_t_7); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = __Pyx_PyInt_As_unsigned_PY_LONG_LONG(__pyx_t_6); if (unlikely((__pyx_t_8 == (unsigned PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_v_next_random = __pyx_t_8; goto __pyx_L4; } __pyx_L4:; - /* "trunk/gensim/models/word2vec_inner.pyx":702 + /* "trunk/gensim/models/word2vec_inner.pyx":374 * * # convert Python structures to primitive types, so we can release the GIL * work = np.PyArray_DATA(_work) # <<<<<<<<<<<<<< * neu1 = np.PyArray_DATA(_neu1) * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) */ - if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 702; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_v__work) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__work, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 374; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_work = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__work))); - /* "trunk/gensim/models/word2vec_inner.pyx":703 + /* "trunk/gensim/models/word2vec_inner.pyx":375 * # convert Python structures to primitive types, so we can release the GIL * work = np.PyArray_DATA(_work) * neu1 = np.PyArray_DATA(_neu1) # <<<<<<<<<<<<<< * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) * */ - if (!(likely(((__pyx_v__neu1) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__neu1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 703; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_v__neu1) == Py_None) || likely(__Pyx_TypeTest(__pyx_v__neu1, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 375; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_neu1 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t *)PyArray_DATA(((PyArrayObject *)__pyx_v__neu1))); - /* "trunk/gensim/models/word2vec_inner.pyx":704 + /* "trunk/gensim/models/word2vec_inner.pyx":376 * work = np.PyArray_DATA(_work) * neu1 = np.PyArray_DATA(_neu1) * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) # <<<<<<<<<<<<<< * * for i in range(sentence_len): */ - __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 704; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Length(__pyx_v_sentence); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 376; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_9 = 10000; if (((__pyx_t_5 < __pyx_t_9) != 0)) { __pyx_t_10 = __pyx_t_5; @@ -6415,7 +4044,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } __pyx_v_sentence_len = ((int)__pyx_t_10); - /* "trunk/gensim/models/word2vec_inner.pyx":706 + /* "trunk/gensim/models/word2vec_inner.pyx":378 * sentence_len = min(MAX_SENTENCE_LEN, len(sentence)) * * for i in range(sentence_len): # <<<<<<<<<<<<<< @@ -6426,19 +4055,19 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { __pyx_v_i = __pyx_t_11; - /* "trunk/gensim/models/word2vec_inner.pyx":707 + /* "trunk/gensim/models/word2vec_inner.pyx":379 * * for i in range(sentence_len): * word = sentence[i] # <<<<<<<<<<<<<< * if word is None: * codelens[i] = 0 */ - __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 707; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_sentence, __pyx_v_i, int, 1, __Pyx_PyInt_From_int, 0, 0, 0); if (unlikely(__pyx_t_6 == NULL)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 379; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __Pyx_GOTREF(__pyx_t_6); __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":708 + /* "trunk/gensim/models/word2vec_inner.pyx":380 * for i in range(sentence_len): * word = sentence[i] * if word is None: # <<<<<<<<<<<<<< @@ -6449,7 +4078,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_12 = (__pyx_t_4 != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":709 + /* "trunk/gensim/models/word2vec_inner.pyx":381 * word = sentence[i] * if word is None: * codelens[i] = 0 # <<<<<<<<<<<<<< @@ -6461,20 +4090,20 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":711 + /* "trunk/gensim/models/word2vec_inner.pyx":383 * codelens[i] = 0 * else: * indexes[i] = word.index # <<<<<<<<<<<<<< * if hs: * codelens[i] = len(word.code) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 711; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_index); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 383; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 711; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_t_6); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 383; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; (__pyx_v_indexes[__pyx_v_i]) = __pyx_t_13; - /* "trunk/gensim/models/word2vec_inner.pyx":712 + /* "trunk/gensim/models/word2vec_inner.pyx":384 * else: * indexes[i] = word.index * if hs: # <<<<<<<<<<<<<< @@ -6484,49 +4113,49 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_12 = (__pyx_v_hs != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":713 + /* "trunk/gensim/models/word2vec_inner.pyx":385 * indexes[i] = word.index * if hs: * codelens[i] = len(word.code) # <<<<<<<<<<<<<< * codes[i] = np.PyArray_DATA(word.code) * points[i] = np.PyArray_DATA(word.point) */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 713; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 385; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_10 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 713; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_10 = PyObject_Length(__pyx_t_6); if (unlikely(__pyx_t_10 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 385; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; (__pyx_v_codelens[__pyx_v_i]) = ((int)__pyx_t_10); - /* "trunk/gensim/models/word2vec_inner.pyx":714 + /* "trunk/gensim/models/word2vec_inner.pyx":386 * if hs: * codelens[i] = len(word.code) * codes[i] = np.PyArray_DATA(word.code) # <<<<<<<<<<<<<< * points[i] = np.PyArray_DATA(word.point) * else: */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 714; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_code); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 386; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 714; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 386; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_codes[__pyx_v_i]) = ((__pyx_t_5numpy_uint8_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":715 + /* "trunk/gensim/models/word2vec_inner.pyx":387 * codelens[i] = len(word.code) * codes[i] = np.PyArray_DATA(word.code) * points[i] = np.PyArray_DATA(word.point) # <<<<<<<<<<<<<< * else: * codelens[i] = 1 */ - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 715; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_word, __pyx_n_s_point); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 387; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 715; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_ptype_5numpy_ndarray))))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 387; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_points[__pyx_v_i]) = ((__pyx_t_5numpy_uint32_t *)PyArray_DATA(((PyArrayObject *)__pyx_t_6))); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; goto __pyx_L8; } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":717 + /* "trunk/gensim/models/word2vec_inner.pyx":389 * points[i] = np.PyArray_DATA(word.point) * else: * codelens[i] = 1 # <<<<<<<<<<<<<< @@ -6537,7 +4166,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } __pyx_L8:; - /* "trunk/gensim/models/word2vec_inner.pyx":718 + /* "trunk/gensim/models/word2vec_inner.pyx":390 * else: * codelens[i] = 1 * result += 1 # <<<<<<<<<<<<<< @@ -6549,7 +4178,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_L7:; } - /* "trunk/gensim/models/word2vec_inner.pyx":720 + /* "trunk/gensim/models/word2vec_inner.pyx":392 * result += 1 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< @@ -6557,17 +4186,17 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc * */ __pyx_t_2 = 0; - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_random); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_randint); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_window); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_sentence_len); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_14); __pyx_t_15 = NULL; __pyx_t_10 = 0; @@ -6581,7 +4210,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_10 = 1; } } - __pyx_t_16 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_16 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_15) { PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_15); __Pyx_GIVEREF(__pyx_t_15); __pyx_t_15 = NULL; @@ -6595,7 +4224,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __Pyx_GIVEREF(__pyx_t_14); __pyx_t_1 = 0; __pyx_t_14 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -6603,9 +4232,9 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); __pyx_t_10 = 0; __pyx_t_17 = NULL; } else { - __pyx_t_10 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_10 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_17 = Py_TYPE(__pyx_t_7)->tp_iternext; if (unlikely(!__pyx_t_17)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; for (;;) { @@ -6613,16 +4242,16 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc if (likely(PyList_CheckExact(__pyx_t_7))) { if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_7)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } else { if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_7)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PySequence_ITEM(__pyx_t_7, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif } } else { @@ -6631,7 +4260,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 720; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 392; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } break; } @@ -6642,17 +4271,17 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_v_i = __pyx_t_2; __pyx_t_2 = (__pyx_t_2 + 1); - /* "trunk/gensim/models/word2vec_inner.pyx":721 + /* "trunk/gensim/models/word2vec_inner.pyx":393 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): * reduced_windows[i] = item # <<<<<<<<<<<<<< * * # release GIL & train on the sentence */ - __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 721; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_13 = __Pyx_PyInt_As_npy_uint32(__pyx_v_item); if (unlikely((__pyx_t_13 == (npy_uint32)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 393; __pyx_clineno = __LINE__; goto __pyx_L1_error;} (__pyx_v_reduced_windows[__pyx_v_i]) = __pyx_t_13; - /* "trunk/gensim/models/word2vec_inner.pyx":720 + /* "trunk/gensim/models/word2vec_inner.pyx":392 * result += 1 * # single randint() call avoids a big thread-sync slowdown * for i, item in enumerate(np.random.randint(0, window, sentence_len)): # <<<<<<<<<<<<<< @@ -6662,7 +4291,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":724 + /* "trunk/gensim/models/word2vec_inner.pyx":396 * * # release GIL & train on the sentence * with nogil: # <<<<<<<<<<<<<< @@ -6676,7 +4305,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc #endif /*try:*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":725 + /* "trunk/gensim/models/word2vec_inner.pyx":397 * # release GIL & train on the sentence * with nogil: * for i in range(sentence_len): # <<<<<<<<<<<<<< @@ -6687,7 +4316,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_2; __pyx_t_11+=1) { __pyx_v_i = __pyx_t_11; - /* "trunk/gensim/models/word2vec_inner.pyx":726 + /* "trunk/gensim/models/word2vec_inner.pyx":398 * with nogil: * for i in range(sentence_len): * if codelens[i] == 0: # <<<<<<<<<<<<<< @@ -6697,7 +4326,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_12 = (((__pyx_v_codelens[__pyx_v_i]) == 0) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":727 + /* "trunk/gensim/models/word2vec_inner.pyx":399 * for i in range(sentence_len): * if codelens[i] == 0: * continue # <<<<<<<<<<<<<< @@ -6707,7 +4336,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc goto __pyx_L14_continue; } - /* "trunk/gensim/models/word2vec_inner.pyx":728 + /* "trunk/gensim/models/word2vec_inner.pyx":400 * if codelens[i] == 0: * continue * j = i - window + reduced_windows[i] # <<<<<<<<<<<<<< @@ -6716,7 +4345,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc */ __pyx_v_j = ((__pyx_v_i - __pyx_v_window) + (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/word2vec_inner.pyx":729 + /* "trunk/gensim/models/word2vec_inner.pyx":401 * continue * j = i - window + reduced_windows[i] * if j < 0: # <<<<<<<<<<<<<< @@ -6726,7 +4355,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_12 = ((__pyx_v_j < 0) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":730 + /* "trunk/gensim/models/word2vec_inner.pyx":402 * j = i - window + reduced_windows[i] * if j < 0: * j = 0 # <<<<<<<<<<<<<< @@ -6738,7 +4367,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } __pyx_L17:; - /* "trunk/gensim/models/word2vec_inner.pyx":731 + /* "trunk/gensim/models/word2vec_inner.pyx":403 * if j < 0: * j = 0 * k = i + window + 1 - reduced_windows[i] # <<<<<<<<<<<<<< @@ -6747,7 +4376,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc */ __pyx_v_k = (((__pyx_v_i + __pyx_v_window) + 1) - (__pyx_v_reduced_windows[__pyx_v_i])); - /* "trunk/gensim/models/word2vec_inner.pyx":732 + /* "trunk/gensim/models/word2vec_inner.pyx":404 * j = 0 * k = i + window + 1 - reduced_windows[i] * if k > sentence_len: # <<<<<<<<<<<<<< @@ -6757,58 +4386,58 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc __pyx_t_12 = ((__pyx_v_k > __pyx_v_sentence_len) != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":733 + /* "trunk/gensim/models/word2vec_inner.pyx":405 * k = i + window + 1 - reduced_windows[i] * if k > sentence_len: * k = sentence_len # <<<<<<<<<<<<<< * if hs: - * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) + * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) */ __pyx_v_k = __pyx_v_sentence_len; goto __pyx_L18; } __pyx_L18:; - /* "trunk/gensim/models/word2vec_inner.pyx":734 + /* "trunk/gensim/models/word2vec_inner.pyx":406 * if k > sentence_len: * k = sentence_len * if hs: # <<<<<<<<<<<<<< - * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) + * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) * if negative: */ __pyx_t_12 = (__pyx_v_hs != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":735 + /* "trunk/gensim/models/word2vec_inner.pyx":407 * k = sentence_len * if hs: - * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) # <<<<<<<<<<<<<< + * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) # <<<<<<<<<<<<<< * if negative: - * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random) + * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random, word_locks) */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), __pyx_v_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, __pyx_v_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean); + __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs((__pyx_v_points[__pyx_v_i]), (__pyx_v_codes[__pyx_v_i]), __pyx_v_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1, __pyx_v_size, __pyx_v_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean, __pyx_v_word_locks); goto __pyx_L19; } __pyx_L19:; - /* "trunk/gensim/models/word2vec_inner.pyx":736 + /* "trunk/gensim/models/word2vec_inner.pyx":408 * if hs: - * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) + * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) * if negative: # <<<<<<<<<<<<<< - * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random) + * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random, word_locks) * */ __pyx_t_12 = (__pyx_v_negative != 0); if (__pyx_t_12) { - /* "trunk/gensim/models/word2vec_inner.pyx":737 - * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) + /* "trunk/gensim/models/word2vec_inner.pyx":409 + * fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) * if negative: - * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random) # <<<<<<<<<<<<<< + * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random, word_locks) # <<<<<<<<<<<<<< * * return result */ - __pyx_v_next_random = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, __pyx_v_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean, __pyx_v_next_random); + __pyx_v_next_random = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg(__pyx_v_negative, __pyx_v_table, __pyx_v_table_len, __pyx_v_codelens, __pyx_v_neu1, __pyx_v_syn0, __pyx_v_syn1neg, __pyx_v_size, __pyx_v_indexes, __pyx_v__alpha, __pyx_v_work, __pyx_v_i, __pyx_v_j, __pyx_v_k, __pyx_v_cbow_mean, __pyx_v_next_random, __pyx_v_word_locks); goto __pyx_L20; } __pyx_L20:; @@ -6816,7 +4445,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } } - /* "trunk/gensim/models/word2vec_inner.pyx":724 + /* "trunk/gensim/models/word2vec_inner.pyx":396 * * # release GIL & train on the sentence * with nogil: # <<<<<<<<<<<<<< @@ -6834,21 +4463,21 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc } } - /* "trunk/gensim/models/word2vec_inner.pyx":739 - * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random) + /* "trunk/gensim/models/word2vec_inner.pyx":411 + * next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random, word_locks) * * return result # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 739; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = __Pyx_PyInt_From_long(__pyx_v_result); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 411; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __pyx_r = __pyx_t_7; __pyx_t_7 = 0; goto __pyx_L0; - /* "trunk/gensim/models/word2vec_inner.pyx":661 + /* "trunk/gensim/models/word2vec_inner.pyx":332 * * * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): # <<<<<<<<<<<<<< @@ -6874,7 +4503,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_2train_sentenc return __pyx_r; } -/* "trunk/gensim/models/word2vec_inner.pyx":742 +/* "trunk/gensim/models/word2vec_inner.pyx":414 * * * def init(): # <<<<<<<<<<<<<< @@ -6913,7 +4542,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U int __pyx_t_4; __Pyx_RefNannySetupContext("init", 0); - /* "trunk/gensim/models/word2vec_inner.pyx":754 + /* "trunk/gensim/models/word2vec_inner.pyx":424 * * cdef int i * cdef float *x = [10.0] # <<<<<<<<<<<<<< @@ -6923,7 +4552,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U __pyx_t_1[0] = ((float)10.0); __pyx_v_x = __pyx_t_1; - /* "trunk/gensim/models/word2vec_inner.pyx":755 + /* "trunk/gensim/models/word2vec_inner.pyx":425 * cdef int i * cdef float *x = [10.0] * cdef float *y = [0.01] # <<<<<<<<<<<<<< @@ -6933,7 +4562,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U __pyx_t_2[0] = ((float)0.01); __pyx_v_y = __pyx_t_2; - /* "trunk/gensim/models/word2vec_inner.pyx":756 + /* "trunk/gensim/models/word2vec_inner.pyx":426 * cdef float *x = [10.0] * cdef float *y = [0.01] * cdef float expected = 0.1 # <<<<<<<<<<<<<< @@ -6942,7 +4571,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U */ __pyx_v_expected = ((float)0.1); - /* "trunk/gensim/models/word2vec_inner.pyx":757 + /* "trunk/gensim/models/word2vec_inner.pyx":427 * cdef float *y = [0.01] * cdef float expected = 0.1 * cdef int size = 1 # <<<<<<<<<<<<<< @@ -6951,7 +4580,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U */ __pyx_v_size = 1; - /* "trunk/gensim/models/word2vec_inner.pyx":762 + /* "trunk/gensim/models/word2vec_inner.pyx":432 * * # build the sigmoid table * for i in range(EXP_TABLE_SIZE): # <<<<<<<<<<<<<< @@ -6961,7 +4590,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U for (__pyx_t_3 = 0; __pyx_t_3 < 1000; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "trunk/gensim/models/word2vec_inner.pyx":763 + /* "trunk/gensim/models/word2vec_inner.pyx":433 * # build the sigmoid table * for i in range(EXP_TABLE_SIZE): * EXP_TABLE[i] = exp((i / EXP_TABLE_SIZE * 2 - 1) * MAX_EXP) # <<<<<<<<<<<<<< @@ -6970,7 +4599,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U */ (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[__pyx_v_i]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)exp(((((__pyx_v_i / ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)1000)) * 2.0) - 1.0) * 6.0))); - /* "trunk/gensim/models/word2vec_inner.pyx":764 + /* "trunk/gensim/models/word2vec_inner.pyx":434 * for i in range(EXP_TABLE_SIZE): * EXP_TABLE[i] = exp((i / EXP_TABLE_SIZE * 2 - 1) * MAX_EXP) * EXP_TABLE[i] = (EXP_TABLE[i] / (EXP_TABLE[i] + 1)) # <<<<<<<<<<<<<< @@ -6980,7 +4609,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U (__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[__pyx_v_i]) = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)((__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[__pyx_v_i]) / ((__pyx_v_5trunk_6gensim_6models_14word2vec_inner_EXP_TABLE[__pyx_v_i]) + 1.0))); } - /* "trunk/gensim/models/word2vec_inner.pyx":767 + /* "trunk/gensim/models/word2vec_inner.pyx":437 * * # check whether sdot returns double or float * d_res = dsdot(&size, x, &ONE, y, &ONE) # <<<<<<<<<<<<<< @@ -6989,67 +4618,49 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U */ __pyx_v_d_res = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot((&__pyx_v_size), __pyx_v_x, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE), __pyx_v_y, (&__pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE)); - /* "trunk/gensim/models/word2vec_inner.pyx":768 + /* "trunk/gensim/models/word2vec_inner.pyx":438 * # check whether sdot returns double or float * d_res = dsdot(&size, x, &ONE, y, &ONE) * p_res = &d_res # <<<<<<<<<<<<<< * if (abs(d_res - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence0_sg_hs + * our_dot = our_dot_double */ __pyx_v_p_res = ((float *)(&__pyx_v_d_res)); - /* "trunk/gensim/models/word2vec_inner.pyx":769 + /* "trunk/gensim/models/word2vec_inner.pyx":439 * d_res = dsdot(&size, x, &ONE, y, &ONE) * p_res = &d_res * if (abs(d_res - expected) < 0.0001): # <<<<<<<<<<<<<< - * fast_sentence_sg_hs = fast_sentence0_sg_hs - * fast_sentence_sg_neg = fast_sentence0_sg_neg + * our_dot = our_dot_double + * our_saxpy = saxpy */ __pyx_t_4 = ((fabs((__pyx_v_d_res - __pyx_v_expected)) < 0.0001) != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":770 + /* "trunk/gensim/models/word2vec_inner.pyx":440 * p_res = &d_res * if (abs(d_res - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence0_sg_hs # <<<<<<<<<<<<<< - * fast_sentence_sg_neg = fast_sentence0_sg_neg - * fast_sentence_cbow_hs = fast_sentence0_cbow_hs - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_hs; - - /* "trunk/gensim/models/word2vec_inner.pyx":771 - * if (abs(d_res - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence0_sg_hs - * fast_sentence_sg_neg = fast_sentence0_sg_neg # <<<<<<<<<<<<<< - * fast_sentence_cbow_hs = fast_sentence0_cbow_hs - * fast_sentence_cbow_neg = fast_sentence0_cbow_neg - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_sg_neg; - - /* "trunk/gensim/models/word2vec_inner.pyx":772 - * fast_sentence_sg_hs = fast_sentence0_sg_hs - * fast_sentence_sg_neg = fast_sentence0_sg_neg - * fast_sentence_cbow_hs = fast_sentence0_cbow_hs # <<<<<<<<<<<<<< - * fast_sentence_cbow_neg = fast_sentence0_cbow_neg + * our_dot = our_dot_double # <<<<<<<<<<<<<< + * our_saxpy = saxpy * return 0 # double */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_hs; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_double; - /* "trunk/gensim/models/word2vec_inner.pyx":773 - * fast_sentence_sg_neg = fast_sentence0_sg_neg - * fast_sentence_cbow_hs = fast_sentence0_cbow_hs - * fast_sentence_cbow_neg = fast_sentence0_cbow_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":441 + * if (abs(d_res - expected) < 0.0001): + * our_dot = our_dot_double + * our_saxpy = saxpy # <<<<<<<<<<<<<< * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence0_cbow_neg; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy; - /* "trunk/gensim/models/word2vec_inner.pyx":774 - * fast_sentence_cbow_hs = fast_sentence0_cbow_hs - * fast_sentence_cbow_neg = fast_sentence0_cbow_neg + /* "trunk/gensim/models/word2vec_inner.pyx":442 + * our_dot = our_dot_double + * our_saxpy = saxpy * return 0 # double # <<<<<<<<<<<<<< * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence1_sg_hs + * our_dot = our_dot_float */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_int_0); @@ -7057,55 +4668,37 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U goto __pyx_L0; } - /* "trunk/gensim/models/word2vec_inner.pyx":775 - * fast_sentence_cbow_neg = fast_sentence0_cbow_neg + /* "trunk/gensim/models/word2vec_inner.pyx":443 + * our_saxpy = saxpy * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): # <<<<<<<<<<<<<< - * fast_sentence_sg_hs = fast_sentence1_sg_hs - * fast_sentence_sg_neg = fast_sentence1_sg_neg + * our_dot = our_dot_float + * our_saxpy = saxpy */ __pyx_t_4 = ((fabsf(((__pyx_v_p_res[0]) - __pyx_v_expected)) < 0.0001) != 0); if (__pyx_t_4) { - /* "trunk/gensim/models/word2vec_inner.pyx":776 + /* "trunk/gensim/models/word2vec_inner.pyx":444 * return 0 # double * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence1_sg_hs # <<<<<<<<<<<<<< - * fast_sentence_sg_neg = fast_sentence1_sg_neg - * fast_sentence_cbow_hs = fast_sentence1_cbow_hs - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_hs; - - /* "trunk/gensim/models/word2vec_inner.pyx":777 - * elif (abs(p_res[0] - expected) < 0.0001): - * fast_sentence_sg_hs = fast_sentence1_sg_hs - * fast_sentence_sg_neg = fast_sentence1_sg_neg # <<<<<<<<<<<<<< - * fast_sentence_cbow_hs = fast_sentence1_cbow_hs - * fast_sentence_cbow_neg = fast_sentence1_cbow_neg - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_sg_neg; - - /* "trunk/gensim/models/word2vec_inner.pyx":778 - * fast_sentence_sg_hs = fast_sentence1_sg_hs - * fast_sentence_sg_neg = fast_sentence1_sg_neg - * fast_sentence_cbow_hs = fast_sentence1_cbow_hs # <<<<<<<<<<<<<< - * fast_sentence_cbow_neg = fast_sentence1_cbow_neg + * our_dot = our_dot_float # <<<<<<<<<<<<<< + * our_saxpy = saxpy * return 1 # float */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_hs; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_float; - /* "trunk/gensim/models/word2vec_inner.pyx":779 - * fast_sentence_sg_neg = fast_sentence1_sg_neg - * fast_sentence_cbow_hs = fast_sentence1_cbow_hs - * fast_sentence_cbow_neg = fast_sentence1_cbow_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":445 + * elif (abs(p_res[0] - expected) < 0.0001): + * our_dot = our_dot_float + * our_saxpy = saxpy # <<<<<<<<<<<<<< * return 1 # float * else: */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence1_cbow_neg; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy = __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy; - /* "trunk/gensim/models/word2vec_inner.pyx":780 - * fast_sentence_cbow_hs = fast_sentence1_cbow_hs - * fast_sentence_cbow_neg = fast_sentence1_cbow_neg + /* "trunk/gensim/models/word2vec_inner.pyx":446 + * our_dot = our_dot_float + * our_saxpy = saxpy * return 1 # float # <<<<<<<<<<<<<< * else: * # neither => use cython loops, no BLAS @@ -7117,45 +4710,27 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U } /*else*/ { - /* "trunk/gensim/models/word2vec_inner.pyx":784 + /* "trunk/gensim/models/word2vec_inner.pyx":450 * # neither => use cython loops, no BLAS * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - * fast_sentence_sg_hs = fast_sentence2_sg_hs # <<<<<<<<<<<<<< - * fast_sentence_sg_neg = fast_sentence2_sg_neg - * fast_sentence_cbow_hs = fast_sentence2_cbow_hs - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_hs; - - /* "trunk/gensim/models/word2vec_inner.pyx":785 - * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - * fast_sentence_sg_hs = fast_sentence2_sg_hs - * fast_sentence_sg_neg = fast_sentence2_sg_neg # <<<<<<<<<<<<<< - * fast_sentence_cbow_hs = fast_sentence2_cbow_hs - * fast_sentence_cbow_neg = fast_sentence2_cbow_neg - */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_sg_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_sg_neg; - - /* "trunk/gensim/models/word2vec_inner.pyx":786 - * fast_sentence_sg_hs = fast_sentence2_sg_hs - * fast_sentence_sg_neg = fast_sentence2_sg_neg - * fast_sentence_cbow_hs = fast_sentence2_cbow_hs # <<<<<<<<<<<<<< - * fast_sentence_cbow_neg = fast_sentence2_cbow_neg + * our_dot = our_dot_noblas # <<<<<<<<<<<<<< + * our_saxpy = our_saxpy_noblas * return 2 */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_hs = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_hs; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_dot = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_dot_noblas; - /* "trunk/gensim/models/word2vec_inner.pyx":787 - * fast_sentence_sg_neg = fast_sentence2_sg_neg - * fast_sentence_cbow_hs = fast_sentence2_cbow_hs - * fast_sentence_cbow_neg = fast_sentence2_cbow_neg # <<<<<<<<<<<<<< + /* "trunk/gensim/models/word2vec_inner.pyx":451 + * # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here + * our_dot = our_dot_noblas + * our_saxpy = our_saxpy_noblas # <<<<<<<<<<<<<< * return 2 * */ - __pyx_v_5trunk_6gensim_6models_14word2vec_inner_fast_sentence_cbow_neg = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_fast_sentence2_cbow_neg; + __pyx_v_5trunk_6gensim_6models_14word2vec_inner_our_saxpy = __pyx_f_5trunk_6gensim_6models_14word2vec_inner_our_saxpy_noblas; - /* "trunk/gensim/models/word2vec_inner.pyx":788 - * fast_sentence_cbow_hs = fast_sentence2_cbow_hs - * fast_sentence_cbow_neg = fast_sentence2_cbow_neg + /* "trunk/gensim/models/word2vec_inner.pyx":452 + * our_dot = our_dot_noblas + * our_saxpy = our_saxpy_noblas * return 2 # <<<<<<<<<<<<<< * * FAST_VERSION = init() # initialize the module @@ -7166,7 +4741,7 @@ static PyObject *__pyx_pf_5trunk_6gensim_6models_14word2vec_inner_4init(CYTHON_U goto __pyx_L0; } - /* "trunk/gensim/models/word2vec_inner.pyx":742 + /* "trunk/gensim/models/word2vec_inner.pyx":414 * * * def init(): # <<<<<<<<<<<<<< @@ -9264,6 +6839,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_snrm2, __pyx_k_snrm2, sizeof(__pyx_k_snrm2), 0, 0, 1, 1}, {&__pyx_n_s_sscal, __pyx_k_sscal, sizeof(__pyx_k_sscal), 0, 0, 1, 1}, {&__pyx_n_s_syn0, __pyx_k_syn0, sizeof(__pyx_k_syn0), 0, 0, 1, 1}, + {&__pyx_n_s_syn0_lockf, __pyx_k_syn0_lockf, sizeof(__pyx_k_syn0_lockf), 0, 0, 1, 1}, {&__pyx_n_s_syn1, __pyx_k_syn1, sizeof(__pyx_k_syn1), 0, 0, 1, 1}, {&__pyx_n_s_syn1neg, __pyx_k_syn1neg, sizeof(__pyx_k_syn1neg), 0, 0, 1, 1}, {&__pyx_n_s_table, __pyx_k_table, sizeof(__pyx_k_table), 0, 0, 1, 1}, @@ -9275,6 +6851,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_n_s_window, __pyx_k_window, sizeof(__pyx_k_window), 0, 0, 1, 1}, {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, + {&__pyx_n_s_word_locks, __pyx_k_word_locks, sizeof(__pyx_k_word_locks), 0, 0, 1, 1}, {&__pyx_n_s_work, __pyx_k_work, sizeof(__pyx_k_work), 0, 0, 1, 1}, {&__pyx_n_s_work_2, __pyx_k_work_2, sizeof(__pyx_k_work_2), 0, 0, 1, 1}, {&__pyx_n_s_x, __pyx_k_x, sizeof(__pyx_k_x), 0, 0, 1, 1}, @@ -9282,8 +6859,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 636; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 307; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 802; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; @@ -9295,31 +6872,31 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "trunk/gensim/models/word2vec_inner.pyx":616 + /* "trunk/gensim/models/word2vec_inner.pyx":287 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_tuple_ = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple_)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple_ = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple_)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - __pyx_tuple__2 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 616; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__2 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 287; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "trunk/gensim/models/word2vec_inner.pyx":699 + /* "trunk/gensim/models/word2vec_inner.pyx":371 * table = (np.PyArray_DATA(model.table)) * table_len = len(model.table) * next_random = (2**24) * np.random.randint(0, 2**24) + np.random.randint(0, 2**24) # <<<<<<<<<<<<<< * * # convert Python structures to primitive types, so we can release the GIL */ - __pyx_tuple__3 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__3 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__3); __Pyx_GIVEREF(__pyx_tuple__3); - __pyx_tuple__4 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 699; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__4 = PyTuple_Pack(2, __pyx_int_0, __pyx_int_16777216); if (unlikely(!__pyx_tuple__4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 371; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__4); __Pyx_GIVEREF(__pyx_tuple__4); @@ -9389,41 +6966,41 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10); - /* "trunk/gensim/models/word2vec_inner.pyx":580 - * return next_random + /* "trunk/gensim/models/word2vec_inner.pyx":250 + * * * def train_sentence_sg(model, sentence, alpha, _work): # <<<<<<<<<<<<<< * cdef int hs = model.hs * cdef int negative = model.negative */ - __pyx_tuple__11 = PyTuple_Pack(28, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_syn0, __pyx_n_s_work_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__11 = PyTuple_Pack(29, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_syn0, __pyx_n_s_word_locks, __pyx_n_s_work_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__11); __Pyx_GIVEREF(__pyx_tuple__11); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(4, 0, 28, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_sg, 580, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(4, 0, 29, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_sg, 250, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "trunk/gensim/models/word2vec_inner.pyx":661 + /* "trunk/gensim/models/word2vec_inner.pyx":332 * * * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): # <<<<<<<<<<<<<< * cdef int hs = model.hs * cdef int negative = model.negative */ - __pyx_tuple__13 = PyTuple_Pack(31, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_neu1, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_cbow_mean, __pyx_n_s_syn0, __pyx_n_s_work_2, __pyx_n_s_neu1_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__13)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__13 = PyTuple_Pack(32, __pyx_n_s_model, __pyx_n_s_sentence, __pyx_n_s_alpha, __pyx_n_s_work, __pyx_n_s_neu1, __pyx_n_s_hs, __pyx_n_s_negative, __pyx_n_s_cbow_mean, __pyx_n_s_syn0, __pyx_n_s_word_locks, __pyx_n_s_work_2, __pyx_n_s_neu1_2, __pyx_n_s_alpha_2, __pyx_n_s_size, __pyx_n_s_codelens, __pyx_n_s_indexes, __pyx_n_s_reduced_windows, __pyx_n_s_sentence_len, __pyx_n_s_window, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_k, __pyx_n_s_result, __pyx_n_s_syn1, __pyx_n_s_points, __pyx_n_s_codes, __pyx_n_s_syn1neg, __pyx_n_s_table, __pyx_n_s_table_len, __pyx_n_s_next_random, __pyx_n_s_word, __pyx_n_s_item); if (unlikely(!__pyx_tuple__13)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(5, 0, 31, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_cbow, 661, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(5, 0, 32, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_train_sentence_cbow, 332, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "trunk/gensim/models/word2vec_inner.pyx":742 + /* "trunk/gensim/models/word2vec_inner.pyx":414 * * * def init(): # <<<<<<<<<<<<<< * """ * Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized */ - __pyx_tuple__15 = PyTuple_Pack(7, __pyx_n_s_i, __pyx_n_s_x, __pyx_n_s_y, __pyx_n_s_expected, __pyx_n_s_size, __pyx_n_s_d_res, __pyx_n_s_p_res); if (unlikely(!__pyx_tuple__15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 742; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_tuple__15 = PyTuple_Pack(7, __pyx_n_s_i, __pyx_n_s_x, __pyx_n_s_y, __pyx_n_s_expected, __pyx_n_s_size, __pyx_n_s_d_res, __pyx_n_s_p_res); if (unlikely(!__pyx_tuple__15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 414; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__15); __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__16 = (PyObject*)__Pyx_PyCode_New(0, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_init, 742, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 742; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_codeobj__16 = (PyObject*)__Pyx_PyCode_New(0, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_scratch_Documents_dev2015, __pyx_n_s_init, 414, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 414; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -9589,115 +7166,115 @@ PyMODINIT_FUNC PyInit_word2vec_inner(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_REAL, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 22; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":57 - * int i, int j, int k, int cbow_mean, unsigned long long next_random) nogil + /* "trunk/gensim/models/word2vec_inner.pyx":34 + * ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil * * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x # <<<<<<<<<<<<<< * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_scopy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_scopy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_scopy = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_scopy_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":58 + /* "trunk/gensim/models/word2vec_inner.pyx":35 * * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x # <<<<<<<<<<<<<< * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_saxpy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_saxpy); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_saxpy = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_saxpy_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":59 + /* "trunk/gensim/models/word2vec_inner.pyx":36 * cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) # <<<<<<<<<<<<<< * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sdot = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_sdot_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":60 + /* "trunk/gensim/models/word2vec_inner.pyx":37 * cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) # <<<<<<<<<<<<<< * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sdot); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_dsdot = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_dsdot_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":61 + /* "trunk/gensim/models/word2vec_inner.pyx":38 * cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) # <<<<<<<<<<<<<< * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x - * cdef fast_sentence_sg_hs_ptr fast_sentence_sg_hs + * */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_snrm2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_snrm2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_snrm2 = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_snrm2_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":62 + /* "trunk/gensim/models/word2vec_inner.pyx":39 * cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) * cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) * cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x # <<<<<<<<<<<<<< - * cdef fast_sentence_sg_hs_ptr fast_sentence_sg_hs - * cdef fast_sentence_sg_neg_ptr fast_sentence_sg_neg + * + * DEF EXP_TABLE_SIZE = 1000 */ - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_fblas); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sscal); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_sscal); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_cpointer); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_5trunk_6gensim_6models_14word2vec_inner_sscal = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_sscal_ptr)PyCObject_AsVoidPtr(__pyx_t_1)); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":73 + /* "trunk/gensim/models/word2vec_inner.pyx":46 * cdef REAL_t[EXP_TABLE_SIZE] EXP_TABLE * * cdef int ONE = 1 # <<<<<<<<<<<<<< @@ -9706,57 +7283,57 @@ PyMODINIT_FUNC PyInit_word2vec_inner(void) */ __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONE = 1; - /* "trunk/gensim/models/word2vec_inner.pyx":74 + /* "trunk/gensim/models/word2vec_inner.pyx":47 * * cdef int ONE = 1 * cdef REAL_t ONEF = 1.0 # <<<<<<<<<<<<<< * - * cdef void fast_sentence0_sg_hs( + * # function implementations swapped based on BLAS detected */ __pyx_v_5trunk_6gensim_6models_14word2vec_inner_ONEF = ((__pyx_t_5trunk_6gensim_6models_14word2vec_inner_REAL_t)1.0); - /* "trunk/gensim/models/word2vec_inner.pyx":580 - * return next_random + /* "trunk/gensim/models/word2vec_inner.pyx":250 + * * * def train_sentence_sg(model, sentence, alpha, _work): # <<<<<<<<<<<<<< * cdef int hs = model.hs * cdef int negative = model.negative */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_1train_sentence_sg, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_1train_sentence_sg, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_sg, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 580; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_sg, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 250; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":661 + /* "trunk/gensim/models/word2vec_inner.pyx":332 * * * def train_sentence_cbow(model, sentence, alpha, _work, _neu1): # <<<<<<<<<<<<<< * cdef int hs = model.hs * cdef int negative = model.negative */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_3train_sentence_cbow, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_3train_sentence_cbow, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_cbow, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 661; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s_train_sentence_cbow, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 332; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":742 + /* "trunk/gensim/models/word2vec_inner.pyx":414 * * * def init(): # <<<<<<<<<<<<<< * """ * Precompute function `sigmoid(x) = 1 / (1 + exp(-x))`, for x values discretized */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_5init, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 742; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5trunk_6gensim_6models_14word2vec_inner_5init, NULL, __pyx_n_s_trunk_gensim_models_word2vec_inn); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 414; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_init, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 742; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s_init, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 414; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "trunk/gensim/models/word2vec_inner.pyx":790 + /* "trunk/gensim/models/word2vec_inner.pyx":454 * return 2 * * FAST_VERSION = init() # initialize the module # <<<<<<<<<<<<<< */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_init); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 790; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_init); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 454; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -9769,14 +7346,14 @@ PyMODINIT_FUNC PyInit_word2vec_inner(void) } } if (__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 790; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 454; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 790; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 454; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FAST_VERSION, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 790; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FAST_VERSION, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 454; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "trunk/gensim/models/word2vec_inner.pyx":1 diff --git a/gensim/models/word2vec_inner.pyx b/gensim/models/word2vec_inner.pyx index 4b35472149..107a66f744 100755 --- a/gensim/models/word2vec_inner.pyx +++ b/gensim/models/word2vec_inner.pyx @@ -31,39 +31,12 @@ ctypedef double (*dsdot_ptr) (const int *N, const float *X, const int *incX, con ctypedef double (*snrm2_ptr) (const int *N, const float *X, const int *incX) nogil ctypedef void (*sscal_ptr) (const int *N, const float *alpha, const float *X, const int *incX) nogil -ctypedef void (*fast_sentence_sg_hs_ptr) ( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work) nogil - -ctypedef unsigned long long (*fast_sentence_sg_neg_ptr) ( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random) nogil - -ctypedef void (*fast_sentence_cbow_hs_ptr) ( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean) nogil - -ctypedef unsigned long long (*fast_sentence_cbow_neg_ptr) ( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random) nogil - cdef scopy_ptr scopy=PyCObject_AsVoidPtr(fblas.scopy._cpointer) # y = x cdef saxpy_ptr saxpy=PyCObject_AsVoidPtr(fblas.saxpy._cpointer) # y += alpha * x cdef sdot_ptr sdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # float = dot(x, y) cdef dsdot_ptr dsdot=PyCObject_AsVoidPtr(fblas.sdot._cpointer) # double = dot(x, y) cdef snrm2_ptr snrm2=PyCObject_AsVoidPtr(fblas.snrm2._cpointer) # sqrt(x^2) cdef sscal_ptr sscal=PyCObject_AsVoidPtr(fblas.sscal._cpointer) # x = alpha * x -cdef fast_sentence_sg_hs_ptr fast_sentence_sg_hs -cdef fast_sentence_sg_neg_ptr fast_sentence_sg_neg -cdef fast_sentence_cbow_hs_ptr fast_sentence_cbow_hs -cdef fast_sentence_cbow_neg_ptr fast_sentence_cbow_neg DEF EXP_TABLE_SIZE = 1000 DEF MAX_EXP = 6 @@ -73,83 +46,65 @@ cdef REAL_t[EXP_TABLE_SIZE] EXP_TABLE cdef int ONE = 1 cdef REAL_t ONEF = 1.0 -cdef void fast_sentence0_sg_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work) nogil: +# function implementations swapped based on BLAS detected +ctypedef REAL_t (*our_dot_ptr) (const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil +ctypedef void (*our_saxpy_ptr) (const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil - cdef long long a, b - cdef long long row1 = word2_index * size, row2 - cdef REAL_t f, g - - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelen): - row2 = word_point[b] * size - f = dsdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) +cdef our_dot_ptr our_dot +cdef our_saxpy_ptr our_saxpy +# for when fblas.sdot returns a double +cdef REAL_t our_dot_double(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + return dsdot(N, X, incX, Y, incY) -cdef void fast_sentence1_sg_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, - REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work) nogil: - - cdef long long a, b - cdef long long row1 = word2_index * size, row2 - cdef REAL_t f, g +# for when fblas.sdot returns a float +cdef REAL_t our_dot_float(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + return sdot(N, X, incX, Y, incY) - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelen): - row2 = word_point[b] * size - f = sdot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) +# for when no blas available +cdef REAL_t our_dot_noblas(const int *N, const float *X, const int *incX, const float *Y, const int *incY) nogil: + # not a true full dot()-implementation: just enough for our cases + cdef int i + cdef REAL_t a + a = 0.0 + for i from 0 <= i < N[0] by 1: + a += X[i] * Y[i] + return a + +# for when no blas available +cdef void our_saxpy_noblas(const int *N, const float *alpha, const float *X, const int *incX, float *Y, const int *incY) nogil: + cdef int i + for i from 0 <= i < N[0] by 1: + Y[i * (incY[0])] = (alpha[0]) * X[i * (incX[0])] + Y[i * (incY[0])] -cdef void fast_sentence2_sg_hs( +cdef void fast_sentence_sg_hs( const np.uint32_t *word_point, const np.uint8_t *word_code, const int codelen, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work) nogil: + const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, REAL_t *word_locks) nogil: cdef long long a, b cdef long long row1 = word2_index * size, row2 cdef REAL_t f, g - for a in range(size): - work[a] = 0.0 + memset(work, 0, size * cython.sizeof(REAL_t)) for b in range(codelen): row2 = word_point[b] * size - f = 0.0 - for a in range(size): - f += syn0[row1 + a] * syn1[row2 + a] + f = our_dot(&size, &syn0[row1], &ONE, &syn1[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (1 - word_code[b] - f) * alpha - for a in range(size): - work[a] += g * syn1[row2 + a] - for a in range(size): - syn1[row2 + a] += g * syn0[row1 + a] - for a in range(size): - syn0[row1 + a] += work[a] + our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1[row2], &ONE) + our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) -cdef unsigned long long fast_sentence0_sg_neg( +cdef unsigned long long fast_sentence_sg_neg( const int negative, np.uint32_t *table, unsigned long long table_len, REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random) nogil: + unsigned long long next_random, REAL_t *word_locks) nogil: cdef long long a cdef long long row1 = word2_index * size, row2 @@ -172,113 +127,28 @@ cdef unsigned long long fast_sentence0_sg_neg( label = 0.0 row2 = target_index * size - f = dsdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + f = our_dot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) + our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + our_saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) + our_saxpy(&size, &word_locks[word2_index], work, &ONE, &syn0[row1], &ONE) return next_random -cdef unsigned long long fast_sentence1_sg_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random) nogil: - - cdef long long a - cdef long long row1 = word2_index * size, row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, label - cdef np.uint32_t target_index - cdef int d - - memset(work, 0, size * cython.sizeof(REAL_t)) - - for d in range(negative+1): - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 - - row2 = target_index * size - f = sdot(&size, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - saxpy(&size, &g, &syn0[row1], &ONE, &syn1neg[row2], &ONE) - - saxpy(&size, &ONEF, work, &ONE, &syn0[row1], &ONE) - - return next_random - -cdef unsigned long long fast_sentence2_sg_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, - REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t word_index, - const np.uint32_t word2_index, const REAL_t alpha, REAL_t *work, - unsigned long long next_random) nogil: - - cdef long long a - cdef long long row1 = word2_index * size, row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, label - cdef np.uint32_t target_index - cdef int d - - for a in range(size): - work[a] = 0.0 - - for d in range(negative+1): - - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 - - row2 = target_index * size - f = 0.0 - for a in range(size): - f += syn0[row1 + a] * syn1neg[row2 + a] - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - for a in range(size): - work[a] += g * syn1neg[row2 + a] - for a in range(size): - syn1neg[row2 + a] += g * syn0[row1 + a] - - for a in range(size): - syn0[row1 + a] += work[a] - - return next_random - -cdef void fast_sentence0_cbow_hs( +cdef void fast_sentence_cbow_hs( const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean) nogil: + int i, int j, int k, int cbow_mean, REAL_t *word_locks) nogil: cdef long long a, b cdef long long row2 - cdef REAL_t f, g, count, inv_count + cdef REAL_t f, g, count, inv_count = 1.0 cdef int m memset(neu1, 0, size * cython.sizeof(REAL_t)) @@ -288,126 +158,43 @@ cdef void fast_sentence0_cbow_hs( continue else: count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - if cbow_mean and count > (0.5): + our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + if count > (0.5): inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - for b in range(codelens[i]): - row2 = word_point[b] * size - f = dsdot(&size, neu1, &ONE, &syn1[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m] * size], &ONE) - -cdef void fast_sentence1_cbow_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean) nogil: - - cdef long long a, b - cdef long long row2 - cdef REAL_t f, g, count, inv_count - cdef int m - - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count , neu1, &ONE) + sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) memset(work, 0, size * cython.sizeof(REAL_t)) for b in range(codelens[i]): row2 = word_point[b] * size - f = sdot(&size, neu1, &ONE, &syn1[row2], &ONE) + f = our_dot(&size, neu1, &ONE, &syn1[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (1 - word_code[b] - f) * alpha - saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) - saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) + our_saxpy(&size, &g, &syn1[row2], &ONE, work, &ONE) + our_saxpy(&size, &g, neu1, &ONE, &syn1[row2], &ONE) - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) + if not cbow_mean: # divide error over summed window vectors + sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) -cdef void fast_sentence2_cbow_hs( - const np.uint32_t *word_point, const np.uint8_t *word_code, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean) nogil: - - cdef long long a, b - cdef long long row2 - cdef REAL_t f, g, count - cdef int m - - for a in range(size): - neu1[a] = 0.0 - count = 0.0 for m in range(j, k): if m == i or codelens[m] == 0: continue else: - count += ONEF - for a in range(size): - neu1[a] += syn0[indexes[m] * size + a] - if cbow_mean and count > (0.5): - for a in range(size): - neu1[a] /= count + our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m] * size], &ONE) - for a in range(size): - work[a] = 0.0 - for b in range(codelens[i]): - row2 = word_point[b] * size - f = 0.0 - for a in range(size): - f += neu1[a] * syn1[row2 + a] - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (1 - word_code[b] - f) * alpha - for a in range(size): - work[a] += g * syn1[row2 + a] - for a in range(size): - syn1[row2 + a] += g * neu1[a] - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - for a in range(size): - syn0[indexes[m] * size + a] += work[a] - -cdef unsigned long long fast_sentence0_cbow_neg( +cdef unsigned long long fast_sentence_cbow_neg( const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random) nogil: + int i, int j, int k, int cbow_mean, unsigned long long next_random, REAL_t *word_locks) nogil: cdef long long a cdef long long row2 cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label + cdef REAL_t f, g, count, inv_count = 1.0, label cdef np.uint32_t target_index, word_index cdef int d, m @@ -420,10 +207,11 @@ cdef unsigned long long fast_sentence0_cbow_neg( continue else: count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - if cbow_mean and count > (0.5): + our_saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) + if count > (0.5): inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) + if cbow_mean: + sscal(&size, &inv_count, neu1, &ONE) # (does this need BLAS-variants like saxpy?) memset(work, 0, size * cython.sizeof(REAL_t)) @@ -439,149 +227,32 @@ cdef unsigned long long fast_sentence0_cbow_neg( label = 0.0 row2 = target_index * size - f = dsdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) + f = our_dot(&size, neu1, &ONE, &syn1neg[row2], &ONE) if f <= -MAX_EXP or f >= MAX_EXP: continue f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - - for m in range(j,k): - if m == i or codelens[m] == 0: - continue - else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) - - return next_random - -cdef unsigned long long fast_sentence1_cbow_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random) nogil: + our_saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) + our_saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) - cdef long long a - cdef long long row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label - cdef np.uint32_t target_index, word_index - cdef int d, m - - word_index = indexes[i] - - memset(neu1, 0, size * cython.sizeof(REAL_t)) - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - saxpy(&size, &ONEF, &syn0[indexes[m] * size], &ONE, neu1, &ONE) - if cbow_mean and count > (0.5): - inv_count = ONEF/count - sscal(&size, &inv_count, neu1, &ONE) - - memset(work, 0, size * cython.sizeof(REAL_t)) - - for d in range(negative+1): - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 - - row2 = target_index * size - f = sdot(&size, neu1, &ONE, &syn1neg[row2], &ONE) - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - saxpy(&size, &g, &syn1neg[row2], &ONE, work, &ONE) - saxpy(&size, &g, neu1, &ONE, &syn1neg[row2], &ONE) + if not cbow_mean: # divide error over summed window vectors + sscal(&size, &inv_count, work, &ONE) # (does this need BLAS-variants like saxpy?) for m in range(j,k): if m == i or codelens[m] == 0: continue else: - saxpy(&size, &ONEF, work, &ONE, &syn0[indexes[m]*size], &ONE) + our_saxpy(&size, &word_locks[indexes[m]], work, &ONE, &syn0[indexes[m]*size], &ONE) return next_random -cdef unsigned long long fast_sentence2_cbow_neg( - const int negative, np.uint32_t *table, unsigned long long table_len, int codelens[MAX_SENTENCE_LEN], - REAL_t *neu1, REAL_t *syn0, REAL_t *syn1neg, const int size, - const np.uint32_t indexes[MAX_SENTENCE_LEN], const REAL_t alpha, REAL_t *work, - int i, int j, int k, int cbow_mean, unsigned long long next_random) nogil: - - cdef long long a - cdef long long row2 - cdef unsigned long long modulo = 281474976710655ULL - cdef REAL_t f, g, count, inv_count, label - cdef np.uint32_t target_index, word_index - cdef int d, m - - word_index = indexes[i] - - for a in range(size): - neu1[a] = 0.0 - count = 0.0 - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - count += ONEF - for a in range(size): - neu1[a] += syn0[indexes[m] * size + a] - if cbow_mean and count > (0.5): - for a in range(size): - neu1[a] /= count - - for a in range(size): - work[a] = 0.0 - - for d in range(negative+1): - if d == 0: - target_index = word_index - label = ONEF - else: - target_index = table[(next_random >> 16) % table_len] - next_random = (next_random * 25214903917ULL + 11) & modulo - if target_index == word_index: - continue - label = 0.0 - - row2 = target_index * size - f = 0.0 - for a in range(size): - f += neu1[a] * syn1neg[row2 + a] - if f <= -MAX_EXP or f >= MAX_EXP: - continue - f = EXP_TABLE[((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))] - g = (label - f) * alpha - for a in range(size): - work[a] += g * syn1neg[row2 + a] - for a in range(size): - syn1neg[row2 + a] += g * neu1[a] - - for m in range(j, k): - if m == i or codelens[m] == 0: - continue - else: - for a in range(size): - syn0[indexes[m] * size + a] += work[a] - - return next_random def train_sentence_sg(model, sentence, alpha, _work): cdef int hs = model.hs cdef int negative = model.negative cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) + cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) cdef REAL_t *work cdef REAL_t _alpha = alpha cdef int size = model.layer1_size @@ -651,9 +322,9 @@ def train_sentence_sg(model, sentence, alpha, _work): if j == i or codelens[j] == 0: continue if hs: - fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work) + fast_sentence_sg_hs(points[i], codes[i], codelens[i], syn0, syn1, size, indexes[j], _alpha, work, word_locks) if negative: - next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random) + next_random = fast_sentence_sg_neg(negative, table, table_len, syn0, syn1neg, size, indexes[i], indexes[j], _alpha, work, next_random, word_locks) return result @@ -664,6 +335,7 @@ def train_sentence_cbow(model, sentence, alpha, _work, _neu1): cdef int cbow_mean = model.cbow_mean cdef REAL_t *syn0 = (np.PyArray_DATA(model.syn0)) + cdef REAL_t *word_locks = (np.PyArray_DATA(model.syn0_lockf)) cdef REAL_t *work cdef REAL_t *neu1 cdef REAL_t _alpha = alpha @@ -732,9 +404,9 @@ def train_sentence_cbow(model, sentence, alpha, _work, _neu1): if k > sentence_len: k = sentence_len if hs: - fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean) + fast_sentence_cbow_hs(points[i], codes[i], codelens, neu1, syn0, syn1, size, indexes, _alpha, work, i, j, k, cbow_mean, word_locks) if negative: - next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random) + next_random = fast_sentence_cbow_neg(negative, table, table_len, codelens, neu1, syn0, syn1neg, size, indexes, _alpha, work, i, j, k, cbow_mean, next_random, word_locks) return result @@ -745,10 +417,8 @@ def init(): into table EXP_TABLE. """ - global fast_sentence_sg_hs - global fast_sentence_sg_neg - global fast_sentence_cbow_hs - global fast_sentence_cbow_neg + global our_dot + global our_saxpy cdef int i cdef float *x = [10.0] @@ -767,24 +437,18 @@ def init(): d_res = dsdot(&size, x, &ONE, y, &ONE) p_res = &d_res if (abs(d_res - expected) < 0.0001): - fast_sentence_sg_hs = fast_sentence0_sg_hs - fast_sentence_sg_neg = fast_sentence0_sg_neg - fast_sentence_cbow_hs = fast_sentence0_cbow_hs - fast_sentence_cbow_neg = fast_sentence0_cbow_neg + our_dot = our_dot_double + our_saxpy = saxpy return 0 # double elif (abs(p_res[0] - expected) < 0.0001): - fast_sentence_sg_hs = fast_sentence1_sg_hs - fast_sentence_sg_neg = fast_sentence1_sg_neg - fast_sentence_cbow_hs = fast_sentence1_cbow_hs - fast_sentence_cbow_neg = fast_sentence1_cbow_neg + our_dot = our_dot_float + our_saxpy = saxpy return 1 # float else: # neither => use cython loops, no BLAS # actually, the BLAS is so messed up we'll probably have segfaulted above and never even reach here - fast_sentence_sg_hs = fast_sentence2_sg_hs - fast_sentence_sg_neg = fast_sentence2_sg_neg - fast_sentence_cbow_hs = fast_sentence2_cbow_hs - fast_sentence_cbow_neg = fast_sentence2_cbow_neg + our_dot = our_dot_noblas + our_saxpy = our_saxpy_noblas return 2 FAST_VERSION = init() # initialize the module diff --git a/gensim/test/test_doc2vec.py b/gensim/test/test_doc2vec.py new file mode 100644 index 0000000000..74c44d14d2 --- /dev/null +++ b/gensim/test/test_doc2vec.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2010 Radim Rehurek +# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html + +""" +Automated tests for checking transformation algorithms (the models package). +""" + +from __future__ import with_statement + +import logging +import unittest +import os +import tempfile +import itertools +import bz2 +from six import iteritems, iterkeys +from six.moves import xrange, zip as izip +from collections import namedtuple + +import numpy as np + +from gensim import utils, matutils +from gensim.models import doc2vec + +module_path = os.path.dirname(__file__) # needed because sample data files are located in the same folder +datapath = lambda fname: os.path.join(module_path, 'test_data', fname) + + +class DocsLeeCorpus(object): + def __init__(self, string_tags=False): + self.string_tags = string_tags + + def _tag(self, i): + return i if not self.string_tags else '_*%d' % i + + def __iter__(self): + with open(datapath('lee_background.cor')) as f: + for i, line in enumerate(f): + yield doc2vec.TaggedDocument(utils.simple_preprocess(line),[self._tag(i)]) + +list_corpus = list(DocsLeeCorpus()) + +sentences = [ + ['human', 'interface', 'computer'], + ['survey', 'user', 'computer', 'system', 'response', 'time'], + ['eps', 'user', 'interface', 'system'], + ['system', 'human', 'system', 'eps'], + ['user', 'response', 'time'], + ['trees'], + ['graph', 'trees'], + ['graph', 'minors', 'trees'], + ['graph', 'minors', 'survey'] + ] + +sentences = [doc2vec.TaggedDocument(words,[i]) for i, words in enumerate(sentences)] + + +def testfile(): + # temporary data will be stored to this file + return os.path.join(tempfile.gettempdir(), 'gensim_doc2vec.tst') + + +class TestDoc2VecModel(unittest.TestCase): + def test_persistence(self): + """Test storing/loading the entire model.""" + model = doc2vec.Doc2Vec(DocsLeeCorpus(), min_count=1) + model.save(testfile()) + self.models_equal(model, doc2vec.Doc2Vec.load(testfile())) + + def test_load_mmap(self): + """Test storing/loading the entire model.""" + model = doc2vec.Doc2Vec(sentences, min_count=1) + + # test storing the internal arrays into separate files + model.save(testfile(), sep_limit=0) + self.models_equal(model, doc2vec.Doc2Vec.load(testfile())) + + # make sure mmaping the arrays back works, too + self.models_equal(model, doc2vec.Doc2Vec.load(testfile(), mmap='r')) + + def test_int_doctags(self): + """Test doc2vec doctag alternatives""" + corpus = DocsLeeCorpus() + + model = doc2vec.Doc2Vec(min_count=1) + model.build_vocab(corpus) + self.assertEqual(len(model.docvecs.doctag_syn0),300) + self.assertEqual(model.docvecs[0].shape,(300,)) + self.assertRaises(KeyError,model.__getitem__,'_*0') + + def test_string_doctags(self): + """Test doc2vec doctag alternatives""" + corpus = DocsLeeCorpus(True) + + model = doc2vec.Doc2Vec(min_count=1) + model.build_vocab(corpus) + self.assertEqual(len(model.docvecs.doctag_syn0),300) + self.assertEqual(model.docvecs[0].shape,(300,)) + self.assertEqual(model.docvecs['_*0'].shape,(300,)) + self.assertTrue(all(model.docvecs['_*0']==model.docvecs[0])) + + def test_empty_errors(self): + # no input => "RuntimeError: you must first build vocabulary before training the model" + self.assertRaises(RuntimeError, doc2vec.Doc2Vec, []) + + # input not empty, but rather completely filtered out + self.assertRaises(RuntimeError, doc2vec.Doc2Vec, list_corpus, min_count=10000) + + def model_sanity(self, model): + """Any non-trivial model on DocsLeeCorpus can pass these sanity checks""" + fire1 = 0 # doc 0 sydney fires + fire2 = 8 # doc 8 sydney fires + tennis1 = 6 # doc 6 tennis + + # inferred vector should be top10 close to bulk-trained one + doc0_inferred = model.infer_vector(list(DocsLeeCorpus())[0].words) + sims_to_infer = model.docvecs.most_similar([doc0_inferred]) + self.assertTrue(fire1 in [match[0] for match in sims_to_infer]) + + # fire8 should be top20 close to fire1 + sims = model.docvecs.most_similar(fire1,topn=20) + self.assertTrue(fire2 in [match[0] for match in sims]) + + # same sims should appear in lookup by vec as by index + doc0_vec = model.docvecs[fire1] + sims2 = model.docvecs.most_similar(positive=[doc0_vec], topn=21) + sims2 = sims2[1:] # ignore first element of sims2, which is doc itself + self.assertEqual(list(zip(*sims))[0], list(zip(*sims2))[0]) # same doc ids + self.assertTrue(np.allclose(list(zip(*sims))[1], list(zip(*sims2))[1])) # close-enough dists + + # tennis doc should be out-of-place among fire news + self.assertEqual(model.docvecs.doesnt_match([fire1, tennis1, fire2]), tennis1) + + # fire docs should be closer than fire-tennis + self.assertTrue(model.docvecs.similarity(fire1,fire2) > model.docvecs.similarity(fire1,tennis1)) + + def test_training(self): + """Test doc2vec training.""" + corpus = DocsLeeCorpus() + model = doc2vec.Doc2Vec(size=100, min_count=2, iter=20) + model.build_vocab(corpus) + self.assertEqual(model.docvecs.doctag_syn0.shape, (300, 100)) + model.train(corpus) + + self.model_sanity(model) + + # build vocab and train in one step; must be the same as above + model2 = doc2vec.Doc2Vec(corpus, size=100, min_count=2, iter=20) + self.models_equal(model, model2) + + def test_dbow_hs(self): + """Test DBOW doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=0, hs=1, negative=0, min_count=2, iter=20) + self.model_sanity(model) + + def test_dmm_hs(self): + """Test DM/mean doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_mean=1, size=24, window=4, hs=1, negative=0, + min_count=2, iter=20) + self.model_sanity(model) + + def test_dms_hs(self): + """Test DM/sum doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_mean=0, size=24, window=4, hs=1, negative=0, + min_count=2, iter=20) + self.model_sanity(model) + + def test_dmc_hs(self): + """Test DM/concatenate doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_concat=1, size=24, window=4, hs=1, negative=0, + min_count=2, iter=20) + self.model_sanity(model) + + def test_dbow_neg(self): + """Test DBOW doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=0, hs=0, negative=10, min_count=2, iter=20) + self.model_sanity(model) + + def test_dmm_neg(self): + """Test DM/mean doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_mean=1, size=24, window=4, hs=0, negative=10, + min_count=2, iter=20) + self.model_sanity(model) + + def test_dms_neg(self): + """Test DM/sum doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_mean=0, size=24, window=4, hs=0, negative=10, + min_count=2, iter=20) + self.model_sanity(model) + + def test_dmc_neg(self): + """Test DM/concatenate doc2vec training.""" + model = doc2vec.Doc2Vec(list_corpus, dm=1, dm_concat=1, size=24, window=4, hs=0, negative=10, + min_count=2, iter=20) + self.model_sanity(model) + + def test_parallel(self): + """Test doc2vec parallel training.""" + if doc2vec.FAST_VERSION < 0: # don't test the plain NumPy version for parallelism (too slow) + return + + corpus = utils.RepeatCorpus(DocsLeeCorpus(), 10000) + + for workers in [2, 4]: + model = doc2vec.Doc2Vec(corpus, workers=workers) + self.model_sanity(model) + + def test_deterministic_hs(self): + """Test doc2vec results identical with identical RNG seed.""" + # hs + model = doc2vec.Doc2Vec(DocsLeeCorpus(), seed=42, workers=1) + model2 = doc2vec.Doc2Vec(DocsLeeCorpus(), seed=42, workers=1) + self.models_equal(model, model2) + + def test_deterministic_neg(self): + """Test doc2vec results identical with identical RNG seed.""" + # neg + model = doc2vec.Doc2Vec(DocsLeeCorpus(), hs=0, negative=3, seed=42, workers=1) + model2 = doc2vec.Doc2Vec(DocsLeeCorpus(), hs=0, negative=3, seed=42, workers=1) + self.models_equal(model, model2) + + def test_deterministic_dmc(self): + """Test doc2vec results identical with identical RNG seed.""" + # bigger, dmc + model = doc2vec.Doc2Vec(DocsLeeCorpus(), dm=1, dm_concat=1, size=24, window=4, hs=1, negative=3, + seed=42, workers=1) + model2 = doc2vec.Doc2Vec(DocsLeeCorpus(), dm=1, dm_concat=1, size=24, window=4, hs=1, negative=3, + seed=42, workers=1) + self.models_equal(model, model2) + + def models_equal(self, model, model2): + # check words/hidden-weights + self.assertEqual(len(model.vocab), len(model2.vocab)) + self.assertTrue(np.allclose(model.syn0, model2.syn0)) + if model.hs: + self.assertTrue(np.allclose(model.syn1, model2.syn1)) + if model.negative: + self.assertTrue(np.allclose(model.syn1neg, model2.syn1neg)) + # check docvecs + self.assertEqual(len(model.docvecs.doctags), len(model2.docvecs.doctags)) + self.assertEqual(len(model.docvecs.index2doctag), len(model2.docvecs.index2doctag)) + self.assertTrue(np.allclose(model.docvecs.doctag_syn0, model2.docvecs.doctag_syn0)) + +#endclass TestDoc2VecModel + +# following code is useful for reproducing paragraph-vectors paper sentiment experiments + +class ConcatenatedDoc2Vec(object): + """ + Concatenation of multiple models for reproducing the Paragraph Vectors paper. + Models must have exactly-matching vocabulary and document IDs. (Models should + be trained separately; this wrapper just returns concatenated results.) + """ + def __init__(self, models): + self.models = models + if hasattr(models[0],'docvecs'): + self.docvecs = ConcatenatedDocvecs([model.docvecs for model in models]) + + def __getitem__(self, token): + return np.concatenate([model[token] for model in self.models]) + + def infer_vector(self, document, alpha=0.1, min_alpha=0.0001, steps=5): + return np.concatenate([model.infer_vector(document,alpha,min_alpha,steps) for model in self.models]) + + def train(self, ignored): + pass # train subcomponents individually + +class ConcatenatedDocvecs(object): + def __init__(self, models): + self.models = models + + def __getitem__(self, token): + return np.concatenate([model[token] for model in self.models]) + + +SentimentDocument = namedtuple('SentimentDocument','words tags split sentiment') + + +def read_su_sentiment_rotten_tomatoes(dirname, lowercase=True): + """ + Read and return documents from the Stanford Sentiment Treebank + corpus (Rotten Tomatoes reviews), from http://nlp.Stanford.edu/sentiment/ + + Initialize the corpus from a given directory, where + http://nlp.stanford.edu/~socherr/stanfordSentimentTreebank.zip + has been expanded. It's not too big, so compose entirely into memory. + """ + logging.info("loading corpus from %s" % dirname) + + # many mangled chars in sentences (datasetSentences.txt) + chars_sst_mangled = ['à', 'á', 'â', 'ã', 'æ', 'ç', 'è', 'é', 'í', + 'í', 'ï', 'ñ', 'ó', 'ô', 'ö', 'û', 'ü'] + sentence_fixups = [(char.encode('utf-8').decode('latin1'), char) for char in chars_sst_mangled] + # more junk, and the replace necessary for sentence-phrase consistency + sentence_fixups.extend([ + ('Â', ''), + ('\xa0', ' '), + ('-LRB-', '('), + ('-RRB-', ')'), + ]) + # only this junk in phrases (dictionary.txt) + phrase_fixups = [('\xa0', ' ')] + + # sentence_id and split are only positive for the full sentences + + # read sentences to temp {sentence -> (id,split) dict, to correlate with dictionary.txt + info_by_sentence = {} + with open(os.path.join(dirname, 'datasetSentences.txt'), 'r') as sentences: + with open(os.path.join(dirname, 'datasetSplit.txt'), 'r') as splits: + next(sentences) # legend + next(splits) # legend + for sentence_line, split_line in izip(sentences, splits): + (id, text) = sentence_line.split('\t') + id = int(id) + text = text.rstrip() + for junk, fix in sentence_fixups: + text = text.replace(junk, fix) + (id2, split_i) = split_line.split(',') + assert id == int(id2) + if text not in info_by_sentence: # discard duplicates + info_by_sentence[text] = (id, int(split_i)) + + # read all phrase text + phrases = [None] * 239232 # known size of phrases + with open(os.path.join(dirname, 'dictionary.txt'), 'r') as phrase_lines: + for line in phrase_lines: + (text, id) = line.split('|') + for junk, fix in phrase_fixups: + text = text.replace(junk, fix) + phrases[int(id)] = text.rstrip() # for 1st pass just string + + SentimentPhrase = namedtuple('SentimentPhrase', SentimentDocument._fields + ('sentence_id',)) + # add sentiment labels, correlate with sentences + with open(os.path.join(dirname, 'sentiment_labels.txt'), 'r') as sentiments: + next(sentiments) # legend + for line in sentiments: + (id, sentiment) = line.split('|') + id = int(id) + sentiment = float(sentiment) + text = phrases[id] + words = text.split() + if lowercase: + words = [word.lower() for word in words] + (sentence_id, split_i) = info_by_sentence.get(text, (None, 0)) + split = [None,'train','test','dev'][split_i] + phrases[id] = SentimentPhrase(words, [id], split, sentiment, sentence_id) + + assert len([phrase for phrase in phrases if phrase.sentence_id is not None]) == len(info_by_sentence) # all + # counts don't match 8544, 2210, 1101 because 13 TRAIN and 1 DEV sentences are duplicates + assert len([phrase for phrase in phrases if phrase.split == 'train']) == 8531 # 'train' + assert len([phrase for phrase in phrases if phrase.split == 'test']) == 2210 # 'test' + assert len([phrase for phrase in phrases if phrase.split == 'dev']) == 1100 # 'dev' + + logging.info("loaded corpus with %i sentences and %i phrases from %s" + % (len(info_by_sentence), len(phrases), dirname)) + + return phrases + + +if __name__ == '__main__': + logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG) + logging.info("using optimization %s" % doc2vec.FAST_VERSION) + unittest.main() diff --git a/gensim/test/test_word2vec.py b/gensim/test/test_word2vec.py index de0c7fb36c..c955827dca 100644 --- a/gensim/test/test_word2vec.py +++ b/gensim/test/test_word2vec.py @@ -149,6 +149,25 @@ def testTraining(self): model2 = word2vec.Word2Vec(sentences, size=2, min_count=1) self.models_equal(model, model2) + + def testLocking(self): + """Test word2vec training doesn't change locked vectors.""" + corpus = LeeCorpus() + # build vocabulary, don't train yet + for sg in range(2): # test both cbow and sg + model = word2vec.Word2Vec(size=4, hs=1, negative=5, min_count=1, sg=sg, window=5) + model.build_vocab(corpus) + + # remember two vectors + locked0 = numpy.copy(model.syn0[0]) + unlocked1 = numpy.copy(model.syn0[1]) + # lock the vector in slot 0 against change + model.syn0_lockf[0] = 0.0 + + model.train(corpus) + self.assertFalse((unlocked1==model.syn0[1]).all()) # unlocked vector should vary + self.assertTrue((locked0==model.syn0[0]).all()) # locked vector should not vary + def testTrainingCbow(self): """Test CBOW word2vec training.""" # to test training, make the corpus larger by repeating its sentences over and over diff --git a/gensim/utils.py b/gensim/utils.py index 9e28d52b6b..fe1c2e9744 100644 --- a/gensim/utils.py +++ b/gensim/utils.py @@ -247,56 +247,79 @@ def load(cls, fname, mmap=None): """ logger.info("loading %s object from %s" % (cls.__name__, fname)) - if fname.endswith('.gz') or fname.endswith('.bz2'): - compress = True - subname = lambda *args: '.'.join([fname] + list(args) + ['npz']) - else: - compress = False - subname = lambda *args: '.'.join([fname] + list(args) + ['npy']) + compress, subname = SaveLoad._adapt_by_suffix(fname) + + obj = unpickle(fname) + obj._load_specials(fname, mmap, compress, subname) + return obj + + + def _load_specials(self, fname, mmap, compress, subname): + """ + Loads any attributes that were stored specially, and gives the same + opportunity to recursively included SaveLoad instances. + """ mmap_error = lambda x, y: IOError( 'Cannot mmap compressed object %s in file %s. ' % (x, y) + 'Use `load(fname, mmap=None)` or uncompress files manually.') - obj = unpickle(fname) - for attrib in getattr(obj, '__numpys', []): + for attrib in getattr(self, '__recursive_saveloads', []): + cfname = '.'.join((fname, attrib)) + logger.info("loading %s recursively from %s.* with mmap=%s" % ( + attrib, cfname, mmap)) + getattr(self, attrib)._load_specials(cfname, mmap, compress, subname) + + for attrib in getattr(self, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % ( - attrib, subname(attrib), mmap)) + attrib, subname(fname, attrib), mmap)) if compress: if mmap: - raise mmap_error(attrib, subname(attrib)) + raise mmap_error(attrib, subname(fname, attrib)) - val = numpy.load(subname(attrib))['val'] + val = numpy.load(subname(fname, attrib))['val'] else: - val = numpy.load(subname(attrib), mmap_mode=mmap) + val = numpy.load(subname(fname, attrib), mmap_mode=mmap) - setattr(obj, attrib, val) + setattr(self, attrib, val) - for attrib in getattr(obj, '__scipys', []): + for attrib in getattr(self, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % ( - attrib, subname(attrib), mmap)) - sparse = unpickle(subname(attrib)) + attrib, subname(fname, attrib), mmap)) + sparse = unpickle(subname(fname, attrib)) if compress: if mmap: - raise mmap_error(attrib, subname(attrib)) + raise mmap_error(attrib, subname(fname, attrib)) - with numpy.load(subname(attrib, 'sparse')) as f: + with numpy.load(subname(fname, attrib, 'sparse')) as f: sparse.data = f['data'] sparse.indptr = f['indptr'] sparse.indices = f['indices'] else: - sparse.data = numpy.load(subname(attrib, 'data'), mmap_mode=mmap) - sparse.indptr = numpy.load(subname(attrib, 'indptr'), mmap_mode=mmap) - sparse.indices = numpy.load(subname(attrib, 'indices'), mmap_mode=mmap) + sparse.data = numpy.load(subname(fname, attrib, 'data'), mmap_mode=mmap) + sparse.indptr = numpy.load(subname(fname, attrib, 'indptr'), mmap_mode=mmap) + sparse.indices = numpy.load(subname(fname, attrib, 'indices'), mmap_mode=mmap) - setattr(obj, attrib, sparse) + setattr(self, attrib, sparse) - for attrib in getattr(obj, '__ignoreds', []): + for attrib in getattr(self, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) - setattr(obj, attrib, None) - return obj + setattr(self, attrib, None) + + + @staticmethod + def _adapt_by_suffix(fname): + """Give appropriate compress setting and filename formula""" + if fname.endswith('.gz') or fname.endswith('.bz2'): + compress = True + subname = lambda *args: '.'.join(list(args) + ['npz']) + else: + compress = False + subname = lambda *args: '.'.join(list(args) + ['npy']) + return (compress, subname) + def _smart_save(self, fname, separately=None, sep_limit=10 * 1024**2, ignore=frozenset(), pickle_protocol=2): @@ -324,14 +347,30 @@ def _smart_save(self, fname, separately=None, sep_limit=10 * 1024**2, "saving %s object under %s, separately %s" % ( self.__class__.__name__, fname, separately)) - if fname.endswith('.gz') or fname.endswith('.bz2'): - compress = True - subname = lambda *args: '.'.join([fname] + list(args) + ['npz']) - else: - compress = False - subname = lambda *args: '.'.join([fname] + list(args) + ['npy']) + compress, subname = SaveLoad._adapt_by_suffix(fname) + + restores = self._save_specials(fname, separately, sep_limit, ignore, pickle_protocol, + compress, subname) + try: + pickle(self, fname, protocol=pickle_protocol) + finally: + # restore attribs handled specially + for obj, asides in restores: + for attrib, val in iteritems(asides): + setattr(obj, attrib, val) + - tmp = {} + def _save_specials(self, fname, separately, sep_limit, ignore, pickle_protocol, compress, subname): + """ + Save aside any attributes that need to be handled separately, including + by recursion any attributes that are themselves SaveLoad instances. + + Returns a list of (obj, {attrib: value, ...}) settings that the caller + should use to restore each object's attributes that were set aside + during the default pickle(). + + """ + asides = {} sparse_matrices = (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix) if separately is None: separately = [] @@ -344,43 +383,52 @@ def _smart_save(self, fname, separately=None, sep_limit=10 * 1024**2, # whatever's in `separately` or `ignore` at this point won't get pickled for attrib in separately + list(ignore): if hasattr(self, attrib): - tmp[attrib] = getattr(self, attrib) + asides[attrib] = getattr(self, attrib) delattr(self, attrib) + recursive_saveloads = [] + restores = [] + for attrib, val in iteritems(self.__dict__): + if isinstance(val, SaveLoad): + recursive_saveloads.append(attrib) + cfname = '.'.join((fname,attrib)) + restores.extend(val._save_specials(cfname, separately, sep_limit, ignore, + pickle_protocol,compress, subname)) + try: numpys, scipys, ignoreds = [], [], [] - for attrib, val in iteritems(tmp): + for attrib, val in iteritems(asides): if isinstance(val, numpy.ndarray) and attrib not in ignore: numpys.append(attrib) logger.info("storing numpy array '%s' to %s" % ( - attrib, subname(attrib))) + attrib, subname(fname, attrib))) if compress: - numpy.savez_compressed(subname(attrib), val=numpy.ascontiguousarray(val)) + numpy.savez_compressed(subname(fname, attrib), val=numpy.ascontiguousarray(val)) else: - numpy.save(subname(attrib), numpy.ascontiguousarray(val)) + numpy.save(subname(fname, attrib), numpy.ascontiguousarray(val)) elif isinstance(val, (scipy.sparse.csr_matrix, scipy.sparse.csc_matrix)) and attrib not in ignore: scipys.append(attrib) logger.info("storing scipy.sparse array '%s' under %s" % ( - attrib, subname(attrib))) + attrib, subname(fname, attrib))) if compress: - numpy.savez_compressed(subname(attrib, 'sparse'), + numpy.savez_compressed(subname(fname, attrib, 'sparse'), data=val.data, indptr=val.indptr, indices=val.indices) else: - numpy.save(subname(attrib, 'data'), val.data) - numpy.save(subname(attrib, 'indptr'), val.indptr) - numpy.save(subname(attrib, 'indices'), val.indices) + numpy.save(subname(fname, attrib, 'data'), val.data) + numpy.save(subname(fname, attrib, 'indptr'), val.indptr) + numpy.save(subname(fname, attrib, 'indices'), val.indices) data, indptr, indices = val.data, val.indptr, val.indices val.data, val.indptr, val.indices = None, None, None try: # store array-less object - pickle(val, subname(attrib), protocol=pickle_protocol) + pickle(val, subname(fname, attrib), protocol=pickle_protocol) finally: val.data, val.indptr, val.indices = data, indptr, indices else: @@ -390,11 +438,14 @@ def _smart_save(self, fname, separately=None, sep_limit=10 * 1024**2, self.__dict__['__numpys'] = numpys self.__dict__['__scipys'] = scipys self.__dict__['__ignoreds'] = ignoreds - pickle(self, fname, protocol=pickle_protocol) - finally: - # restore the attributes - for attrib, val in iteritems(tmp): + self.__dict__['__recursive_saveloads'] = recursive_saveloads + except: + # restore the attributes if exception-interrupted + for attrib, val in iteritems(asides): setattr(self, attrib, val) + raise + return restores + [(self, asides)] + def save(self, fname_or_handle, separately=None, sep_limit=10 * 1024**2, ignore=frozenset(), pickle_protocol=2):