use long long for value multipled by vector_size for > 2^31 indexing

piskvorky · Aug 1, 2020 · 492ce1a · 492ce1a
1 parent 779fe46
commit 492ce1a
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/gensim/models/doc2vec_corpusfile.pyx b/gensim/models/doc2vec_corpusfile.pyx
@@ -162,7 +162,7 @@ def d2v_train_epoch_dbow(
     cdef int sent_idx, idx_start, idx_end
 
     cdef vector[string] doc_words
-    cdef int _doc_tag = start_doctag
+    cdef long long _doc_tag = start_doctag
 
     init_d2v_config(
         &c, model, _alpha, learn_doctags, learn_words, learn_hidden, train_words=train_words,
@@ -302,7 +302,7 @@ def d2v_train_epoch_dm(
     cdef REAL_t count, inv_count = 1.0
 
     cdef vector[string] doc_words
-    cdef int _doc_tag = start_doctag
+    cdef long long _doc_tag = start_doctag
 
     init_d2v_config(
         &c, model, _alpha, learn_doctags, learn_words, learn_hidden, train_words=False,
@@ -455,7 +455,7 @@ def d2v_train_epoch_dm_concat(
     cdef int sent_idx, idx_start, idx_end
 
     cdef vector[string] doc_words
-    cdef int _doc_tag = start_doctag
+    cdef long long _doc_tag = start_doctag
 
     init_d2v_config(
         &c, model, _alpha, learn_doctags, learn_words, learn_hidden, train_words=False,
@@ -469,8 +469,8 @@ def d2v_train_epoch_dm_concat(
             effective_words = 0
 
             doc_words = input_stream.read_sentence()
-            _doc_tag = total_documents
-            c.doctag_len = _doc_tag < c.docvecs_count
+            _doc_tag = total_documents  # WTF? skipping to end no matter what start_doctag passed in?
+            c.doctag_len = _doc_tag < c.docvecs_count  # WTF? if only 1 acceptable as this scalar, odd to set from bool
 
              # skip doc either empty or without expected number of tags
             if doc_words.empty() or c.expected_doctag_len != c.doctag_len: