avoid collisions when decoding bad unicode

gensim/models/_fasttext_bin.py

@@ -35,6 +35,7 @@
 import struct
 
 import numpy as np
+import six
 
 _END_OF_WORD_MARKER = b'\x00'
 
@@ -160,6 +161,16 @@ def _load_vocab(fin, new_format, encoding='utf-8'):
     """
     vocab_size, nwords, nlabels = _struct_unpack(fin, '@3i')
 
+    #
+    # We must use backslashreplace instead of replace or ignore here, because
+    # we must avoid collisions in the decoded word, e.g.
+    # https://github.com/RaRe-Technologies/gensim/issues/2402
+    #
+    # Unfortunately, backslashreplace is only available on Py3.  On Py2, we
+    # can't really do anything to avoid collisions.
+    #
+    errors = 'backslashreplace' if six.PY3 else 'replace'
+
     # Vocab stored by [Dictionary::save](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc)
     if nlabels > 0:
         raise NotImplementedError("Supervised fastText models are not supported")
@@ -182,7 +193,7 @@ def _load_vocab(fin, new_format, encoding='utf-8'):
         try:
             word = word_bytes.decode(encoding)
         except UnicodeDecodeError:
-            word = word_bytes.decode(encoding, errors='ignore')
+            word = word_bytes.decode(encoding, errors=errors)
             logger.error(
                 'failed to decode invalid unicode bytes %r; ignoring invalid characters, using %r',
                 word_bytes, word

gensim/test/test_fasttext.py

@@ -1261,10 +1261,18 @@ def test_bad_unicode(self):
         buf.seek(0)
 
         raw_vocab, vocab_size, nlabels = gensim.models._fasttext_bin._load_vocab(buf, False)
-        expected = {
-            u'英語版ウィキペディアへの投稿はいつでも': 1,
-            u'административно-территориальн': 2,
-        }
+
+        if six.PY3:
+            expected = {
+                u'英語版ウィキペディアへの投稿はいつでも\\xe6': 1,
+                u'административно-территориальн\\xd1': 2,
+            }
+        else:
+            expected = {
+                u'英語版ウィキペディアへの投稿はいつでも�': 1,
+                u'административно-территориальн�': 2,
+            }
+
         self.assertEqual(expected, dict(raw_vocab))
 
         self.assertEqual(vocab_size, 2)