#5043 - Ability to specify token breaking zones when calling tokenizer

- Fix case where there is only a single boundary given
inception-project · Sep 30, 2024 · 3b2bfa6 · 3b2bfa6
1 parent 7022635
commit 3b2bfa6
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 17 deletions.
diff --git a/...on-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java b/...on-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java
@@ -142,22 +142,20 @@ public static boolean isEmpty(int aBegin, int aEnd)
 
     private static int[] sortedZoneBoundaries(CAS aCas, Iterable<? extends AnnotationFS> aZones)
     {
-        int[] sortedZoneBoundaries = null;
+        var zoneBoundaries = new IntArrayList();
 
         if (aZones != null) {
-            var zoneBoundaries = new IntArrayList();
             for (var zone : aZones) {
                 zoneBoundaries.add(zone.getBegin());
                 zoneBoundaries.add(zone.getEnd());
             }
-
-            sortedZoneBoundaries = zoneBoundaries.intStream().distinct().sorted().toArray();
         }
 
-        if (sortedZoneBoundaries == null || sortedZoneBoundaries.length < 2) {
-            sortedZoneBoundaries = new int[] { 0, aCas.getDocumentText().length() };
+        if (zoneBoundaries.size() < 2) {
+            zoneBoundaries.add(0);
+            zoneBoundaries.add(aCas.getDocumentText().length());
         }
 
-        return sortedZoneBoundaries;
+        return zoneBoundaries.intStream().distinct().sorted().toArray();
     }
 }
diff --git a/...ception/export/SegmentationUtilsTest.java → ...n/support/uima/SegmentationUtilsTest.java b/...ception/export/SegmentationUtilsTest.java → ...n/support/uima/SegmentationUtilsTest.java
@@ -15,12 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package de.tudarmstadt.ukp.inception.export;
+package de.tudarmstadt.ukp.inception.support.uima;
 
 import static de.tudarmstadt.ukp.inception.support.uima.SegmentationUtils.splitSentences;
 import static de.tudarmstadt.ukp.inception.support.uima.SegmentationUtils.tokenize;
 import static org.apache.uima.fit.factory.JCasFactory.createText;
-import static org.apache.uima.fit.util.CasUtil.toText;
 import static org.apache.uima.fit.util.JCasUtil.select;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -41,7 +40,8 @@ public void testSplitSentences() throws Exception
 
         splitSentences(jcas.getCas());
 
-        assertThat(toText(select(jcas, Sentence.class))) //
+        assertThat(select(jcas, Sentence.class)) //
+                .extracting(Sentence::getCoveredText) //
                 .containsExactly("I am one.", "I am two.");
     }
 
@@ -54,7 +54,8 @@ public void testSplitSentencesWithZones() throws Exception
 
         splitSentences(jcas.getCas(), jcas.select(Div.class));
 
-        assertThat(toText(select(jcas, Sentence.class))) //
+        assertThat(select(jcas, Sentence.class)) //
+                .extracting(Sentence::getCoveredText) //
                 .containsExactly("Heading", "I am two.");
     }
 
@@ -67,11 +68,15 @@ public void testTokenize() throws Exception
 
         tokenize(jcas.getCas());
 
-        assertThat(toText(select(jcas, Sentence.class))) //
+        assertThat(select(jcas, Sentence.class)) //
+                .extracting(Sentence::getCoveredText) //
                 .containsExactly("i am one.", "i am two.");
 
-        assertThat(toText(select(jcas, Token.class))) //
-                .containsExactly("i", "am", "one", ".", "i", "am", "two", ".");
+        assertThat(select(jcas, Token.class)) //
+                .extracting(Token::getCoveredText) //
+                .containsExactly( //
+                        "i", "am", "one", ".", //
+                        "i", "am", "two", ".");
     }
 
     @Test
@@ -85,10 +90,14 @@ public void testTokenizeWitZones() throws Exception
 
         tokenize(jcas.getCas(), jcas.select(Div.class));
 
-        assertThat(toText(select(jcas, Sentence.class))) //
+        assertThat(select(jcas, Sentence.class)) //
+                .extracting(Sentence::getCoveredText) //
                 .containsExactly("i am one.", "i am two.");
 
-        assertThat(toText(select(jcas, Token.class))) //
-                .containsExactly("i", "a", "m", "one", ".", "i", "a", "m", "t", "wo", ".");
+        assertThat(select(jcas, Token.class)) //
+                .extracting(Token::getCoveredText) //
+                .containsExactly( //
+                        "i", "a", "m", "one", ".", //
+                        "i", "a", "m", "t", "wo", ".");
     }
 }