Loading fastText models using only bin file

gensim/models/wrappers/fasttext.py

@@ -35,11 +35,15 @@
 import numpy as np
 from numpy import float32 as REAL, sqrt, newaxis
 from gensim import utils
-from gensim.models.keyedvectors import KeyedVectors
+from gensim.models.keyedvectors import KeyedVectors, Vocab
 from gensim.models.word2vec import Word2Vec
 
 from six import string_types
 
+from numpy import exp, log, dot, zeros, outer, random, dtype, float32 as REAL,\
+    double, uint32, seterr, array, uint8, vstack, fromstring, sqrt, newaxis,\
+    ndarray, empty, sum as np_sum, prod, ones, ascontiguousarray
+
 logger = logging.getLogger(__name__)
 
 FASTTEXT_FILEFORMAT_MAGIC = 793712314
@@ -224,7 +228,7 @@ def load_word2vec_format(cls, *args, **kwargs):
         return FastTextKeyedVectors.load_word2vec_format(*args, **kwargs)
 
     @classmethod
-    def load_fasttext_format(cls, model_file, encoding='utf8'):
+    def load_fasttext_format(cls, model_file, bin_only = False, encoding='utf8'):
         """
         Load the input-hidden weight matrix from the fast text output files.
 
@@ -237,8 +241,11 @@ def load_fasttext_format(cls, model_file, encoding='utf8'):
 
         """
         model = cls()
-        model.wv = cls.load_word2vec_format('%s.vec' % model_file, encoding=encoding)
-        model.load_binary_data('%s.bin' % model_file, encoding=encoding)
+        if bin_only:
+            model.load_binary_data('%s.bin' % model_file, bin_only, encoding=encoding)
+        else:
+            model.wv = cls.load_word2vec_format('%s.vec' % model_file, encoding=encoding)
+            model.load_binary_data('%s.bin' % model_file, encoding=encoding)
         return model
 
     @classmethod
@@ -251,11 +258,11 @@ def delete_training_files(cls, model_file):
             logger.debug('Training files %s not found when attempting to delete', model_file)
             pass
 
-    def load_binary_data(self, model_binary_file, encoding='utf8'):
+    def load_binary_data(self, model_binary_file, bin_only = False, encoding='utf8'):
         """Loads data from the output binary file created by FastText training"""
         with utils.smart_open(model_binary_file, 'rb') as f:
             self.load_model_params(f)
-            self.load_dict(f, encoding=encoding)
+            self.load_dict(f, bin_only, encoding=encoding)
             self.load_vectors(f)
 
     def load_model_params(self, file_handle):
@@ -281,15 +288,22 @@ def load_model_params(self, file_handle):
         self.wv.max_n = maxn
         self.sample = t
 
-    def load_dict(self, file_handle, encoding='utf8'):
+    def load_dict(self, file_handle, bin_only = False, encoding='utf8'):
         vocab_size, nwords, _ = self.struct_unpack(file_handle, '@3i')
         # Vocab stored by [Dictionary::save](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc)
-        assert len(self.wv.vocab) == nwords, 'mismatch between vocab sizes'
-        assert len(self.wv.vocab) == vocab_size, 'mismatch between vocab sizes'
-        self.struct_unpack(file_handle, '@1q')  # number of tokens
+        if not bin_only:
+            assert len(self.wv.vocab) == nwords, 'mismatch between vocab sizes'
+            if len(self.wv.vocab) != vocab_size:
+                logger.warnings("If you are loading any model other than pretrained vector wiki.fr, ")
+                logger.warnings("Please report to gensim or fastText.")
+        else:
+            self.wv.syn0 = zeros((vocab_size, self.vector_size), dtype=REAL)
+            logger.info("here?")
+            # TO-DO : how to update this
+        ntokens= self.struct_unpack(file_handle, '@1q')
         if self.new_format:
             pruneidx_size, = self.struct_unpack(file_handle, '@q')
-        for i in range(nwords):
+        for i in range(vocab_size):
             word_bytes = b''
             char_byte = file_handle.read(1)
             # Read vocab word
@@ -298,8 +312,24 @@ def load_dict(self, file_handle, encoding='utf8'):
                 char_byte = file_handle.read(1)
             word = word_bytes.decode(encoding)
             count, _ = self.struct_unpack(file_handle, '@qb')
-            assert self.wv.vocab[word].index == i, 'mismatch between gensim word index and fastText word index'
-            self.wv.vocab[word].count = count
+            if bin_only and word not in self.wv.vocab:
+                self.wv.vocab[word] = Vocab(index=i, count=count)
+            elif not bin_only:
+                assert self.wv.vocab[word].index == i, 'mismatch between gensim word index and fastText word index'
+                self.wv.vocab[word].count = count
+
+            if bin_only:
+                #self.wv.syn0[i] = weight  # How to get weight vector for each word ?
+                self.wv.index2word.append(word)
+
+        if bin_only:
+            if self.wv.syn0.shape[0] != len(self.wv.vocab):
+                logger.info(
+                    "duplicate words detected, shrinking matrix size from %i to %i",
+                    self.wv.syn0.shape[0], len(self.wv.vocab)
+                )
+                self.wv.syn0 = ascontiguousarray(result.syn0[: len(self.wv.vocab)])
+            assert (len(self.wv.vocab), self.vector_size) == self.wv.syn0.shape
 
         if self.new_format:
             for j in range(pruneidx_size):

gensim/test/test_fasttext_wrapper.py

@@ -171,6 +171,37 @@ def testLoadFastTextNewFormat(self):
         self.assertEquals(self.test_new_model.wv.min_n, 3)
         self.model_sanity(new_model)
 
+    def testLoadBinOnly(self):
+        """ Test model succesfully loaded from fastText (new format) .bin files only """
+        new_model = fasttext.FastText.load_fasttext_format(self.test_new_model_file, bin_only = True)
+        vocab_size, model_size = 1763, 10
+        self.assertEqual(self.test_new_model.wv.syn0.shape, (vocab_size, model_size))
+        self.assertEqual(len(self.test_new_model.wv.vocab), vocab_size, model_size)
+        self.assertEqual(self.test_new_model.wv.syn0_all.shape, (self.test_new_model.num_ngram_vectors, model_size))
+
+        expected_vec_new = [-0.025627,
+                            -0.11448,
+                             0.18116,
+                            -0.96779,
+                             0.2532,
+                            -0.93224,
+                             0.3929,
+                             0.12679,
+                            -0.19685,
+                            -0.13179]  # obtained using ./fasttext print-word-vectors lee_fasttext_new.bin < queries.txt
+
+        self.assertTrue(numpy.allclose(self.test_new_model["hundred"], expected_vec_new, 0.001))
+        self.assertEquals(self.test_new_model.min_count, 5)
+        self.assertEquals(self.test_new_model.window, 5)
+        self.assertEquals(self.test_new_model.iter, 5)
+        self.assertEquals(self.test_new_model.negative, 5)
+        self.assertEquals(self.test_new_model.sample, 0.0001)
+        self.assertEquals(self.test_new_model.bucket, 1000)
+        self.assertEquals(self.test_new_model.wv.max_n, 6)
+        self.assertEquals(self.test_new_model.wv.min_n, 3)
+        self.model_sanity(new_model)
+
+
     def testLoadModelWithNonAsciiVocab(self):
         """Test loading model with non-ascii words in vocab"""
         model = fasttext.FastText.load_fasttext_format(datapath('non_ascii_fasttext'))