Improve the error when passing an empty sentence to the tokenizer (#40)

* Add clearer error when an empty sentence is passed

* Update changelog

* Add test case

CHANGELOG.md

@@ -17,6 +17,10 @@ Types of changes
 
 ## [Unreleased]
 
+### Changed
+
+- Changed the error when an empty sentence is provided to the tokenizer.
+
 ### Fixed
 
 - No longer override `language` metadata from the dataset if the language was also set manually via `SpanMarkerModelCardData`.

span_marker/tokenizer.py

@@ -206,7 +206,10 @@ def __call__(
         all_labels = []
         all_num_words = []
         for sample_idx, input_ids in enumerate(batch_encoding["input_ids"]):
-            num_words = int(np.nanmax(np.array(batch_encoding.word_ids(sample_idx), dtype=float))) + 1
+            max_word_ids = np.nanmax(np.array(batch_encoding.word_ids(sample_idx), dtype=float))
+            if np.isnan(max_word_ids):
+                raise ValueError("The `SpanMarkerTokenizer` detected an empty sentence, please remove it.")
+            num_words = int(max_word_ids) + 1
             if self.tokenizer.pad_token_id in input_ids:
                 num_tokens = list(input_ids).index(self.tokenizer.pad_token_id)
             else:

tests/test_modeling.py

@@ -202,6 +202,12 @@ def test_predict_where_first_sentence_is_word(finetuned_conll_span_marker_model:
     assert isinstance(outputs[0], list)
 
 
+def test_predict_empty_error(finetuned_conll_span_marker_model: SpanMarkerModel) -> None:
+    model = finetuned_conll_span_marker_model.try_cuda()
+    with pytest.raises(ValueError, match="The `SpanMarkerTokenizer` detected an empty sentence, please remove it."):
+        model.predict(["One Two", "Three Four Five", ""])
+
+
 def test_incorrect_predict_inputs(finetuned_conll_span_marker_model: SpanMarkerModel):
     model = finetuned_conll_span_marker_model.try_cuda()
     with pytest.raises(ValueError, match="could not recognize your input"):