Upgrade morfologik from 1.10.0 to 2.0.1

languagetool-core/pom.xml

@@ -101,6 +101,11 @@
             <artifactId>morfologik-fsa</artifactId>
             <version>${morfologik.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.carrot2</groupId>
+            <artifactId>morfologik-fsa-builders</artifactId>
+            <version>${morfologik.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.carrot2</groupId>
             <artifactId>morfologik-speller</artifactId>
@@ -115,7 +120,7 @@
             <!-- needed for building the Morfologik dict at runtime (for ignore.txt) -->
             <groupId>com.carrotsearch</groupId>
             <artifactId>hppc</artifactId>
-            <version>0.6.1</version>
+            <version>0.7.1</version>
         </dependency>
         <dependency>
             <groupId>net.loomchild</groupId>

languagetool-core/src/main/java/org/languagetool/rules/spelling/morfologik/MorfologikMultiSpeller.java

@@ -18,9 +18,9 @@
  */
 package org.languagetool.rules.spelling.morfologik;
 
-import morfologik.fsa.CFSA2Serializer;
 import morfologik.fsa.FSA;
-import morfologik.fsa.FSABuilder;
+import morfologik.fsa.builders.CFSA2Serializer;
+import morfologik.fsa.builders.FSABuilder;
 import morfologik.stemming.Dictionary;
 import org.jetbrains.annotations.Nullable;
 import org.languagetool.JLanguageTool;
@@ -112,11 +112,14 @@ private Dictionary getDictionary(List<byte[]> lines, String dictPath) throws IOE
       Collections.sort(lines, FSABuilder.LEXICAL_ORDERING);
       FSA fsa = FSABuilder.build(lines);
       ByteArrayOutputStream fsaOutStream = new CFSA2Serializer().serialize(fsa, new ByteArrayOutputStream());
-      ByteArrayInputStream fsaInStream = new ByteArrayInputStream(fsaOutStream.toByteArray());
       String infoFile = dictPath.replace(".dict", ".info");
-      Dictionary dict = Dictionary.readAndClose(fsaInStream, JLanguageTool.getDataBroker().getFromResourceDirAsStream(infoFile));
-      dicPathToDict.put(dictPath, dict);
-      return dict;
+
+      try (InputStream fsaInStream = new ByteArrayInputStream(fsaOutStream.toByteArray());
+           InputStream metadataInStream = JLanguageTool.getDataBroker().getFromResourceDirAsStream(infoFile)) {
+        Dictionary dict = Dictionary.read(fsaInStream, metadataInStream);
+        dicPathToDict.put(dictPath, dict);
+        return dict;
+      }
     }
   }
 

languagetool-core/src/main/java/org/languagetool/rules/spelling/morfologik/MorfologikSpeller.java

@@ -29,7 +29,6 @@
 import org.languagetool.tools.StringTools;
 
 import java.io.IOException;
-import java.nio.charset.CharacterCodingException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -88,12 +87,8 @@ public boolean isMisspelled(String word) {
 
   public List<String> getSuggestions(String word) {
     final List<String> suggestions = new ArrayList<>();
-    try {
-      suggestions.addAll(speller.findReplacements(word));
-      suggestions.addAll(speller.replaceRunOnWords(word));
-    } catch (CharacterCodingException e) {
-      throw new RuntimeException(e);
-    }
+    suggestions.addAll(speller.findReplacements(word));
+    suggestions.addAll(speller.replaceRunOnWords(word));
     // capitalize suggestions if necessary
     if (dictionary.metadata.isConvertingCase() && StringTools.startsWithUppercase(word)) {
       for (int i = 0; i < suggestions.size(); i++) {

languagetool-language-modules/de/src/test/java/org/languagetool/rules/de/GermanSpellerRuleTest.java

@@ -18,9 +18,9 @@
  */
 package org.languagetool.rules.de;
 
-import morfologik.fsa.CFSA2Serializer;
 import morfologik.fsa.FSA;
-import morfologik.fsa.FSABuilder;
+import morfologik.fsa.builders.CFSA2Serializer;
+import morfologik.fsa.builders.FSABuilder;
 import morfologik.speller.Speller;
 import morfologik.stemming.Dictionary;
 import org.junit.Ignore;
@@ -42,9 +42,7 @@
 
 import static junit.framework.TestCase.assertFalse;
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 public class GermanSpellerRuleTest {
 
@@ -321,8 +319,10 @@ private Dictionary getDictionary(List<byte[]> lines, InputStream infoFile) throw
     Collections.sort(lines, FSABuilder.LEXICAL_ORDERING);
     FSA fsa = FSABuilder.build(lines);
     ByteArrayOutputStream fsaOutStream = new CFSA2Serializer().serialize(fsa, new ByteArrayOutputStream());
-    ByteArrayInputStream fsaInStream = new ByteArrayInputStream(fsaOutStream.toByteArray());
-    return Dictionary.readAndClose(fsaInStream, infoFile);
+    try (InputStream fsaInStream = new ByteArrayInputStream(fsaOutStream.toByteArray());
+         InputStream metadataInStream = infoFile) {
+      return Dictionary.read(fsaInStream, metadataInStream);
+    }
   }
 
   private void assertCorrection(HunspellRule rule, String input, String... expectedTerms) throws IOException {

languagetool-standalone/pom.xml

@@ -128,6 +128,12 @@
             <version>${morfologik.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>${commons.cli.version}</version>
+        </dependency>
+
         <dependency>
             <!-- to prevent "Failed to load class org.slf4j.impl.StaticLoggerBinder",
                  see http://www.slf4j.org/codes.html#StaticLoggerBinder -->

languagetool-tools/pom.xml

@@ -64,6 +64,11 @@
             <artifactId>languagetool-core</artifactId>
             <version>${languagetool.version}</version>
         </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>${commons.cli.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.carrot2</groupId>
             <artifactId>morfologik-tools</artifactId>

languagetool-tools/src/main/java/org/languagetool/tools/DictionaryBuilder.java

@@ -18,28 +18,14 @@
  */
 package org.languagetool.tools;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
+import morfologik.tools.FSABuild;
+import morfologik.tools.Launcher;
 import org.jetbrains.annotations.Nullable;
 
-import morfologik.tools.FSABuildTool;
-import morfologik.tools.Launcher;
+import java.io.*;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Create a Morfologik binary dictionary from plain text data.
@@ -109,7 +95,7 @@ protected File buildDict(File tempFile) throws Exception {
 
     String[] buildToolOptions = {"-f", "cfsa2", "-i", tempFile.getAbsolutePath(), "-o", resultFile.getAbsolutePath()};
     System.out.println("Running Morfologik FSABuildTool.main with these options: " + Arrays.toString(buildToolOptions));
-    FSABuildTool.main(buildToolOptions);
+    FSABuild.main(buildToolOptions);
     System.out.println("Done. The binary dictionary has been written to " + resultFile.getAbsolutePath());
     return resultFile;
   }

languagetool-tools/src/main/java/org/languagetool/tools/DictionaryExporter.java

@@ -18,7 +18,8 @@
  */
 package org.languagetool.tools;
 
-import morfologik.tools.FSADumpTool;
+
+import morfologik.tools.FSADump;
 
 import java.io.File;
 
@@ -36,9 +37,11 @@ public static void main(String[] args) throws Exception {
     String filename = args[0];
     String path = new File(filename).getAbsolutePath();
     if (path.contains("hunspell") || path.contains("spelling")) {
-      FSADumpTool.main("--raw-data", "-d", args[0]);
+      String[] options = {"--raw-data", "-d", args[0]};
+      FSADump.main(options);
     } else {
-      FSADumpTool.main("--raw-data", "-x", "-d", args[0]);
+      String[] options = {"--raw-data", "-x", "-d", args[0]};
+      FSADump.main(options);
     }
   }
 

languagetool-wikipedia/pom.xml

@@ -124,7 +124,7 @@
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>
-            <version>1.2</version>
+            <version>${commons.cli.version}</version>
         </dependency>
         <dependency>
             <groupId>org.sweble.wikitext</groupId>

pom.xml

@@ -28,8 +28,9 @@
       <maven.compiler.source>1.8</maven.compiler.source>
       <maven.compiler.target>1.8</maven.compiler.target>
       <maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>
+      <commons.cli.version>1.2</commons.cli.version>
       <junit.version>4.12</junit.version>
-      <morfologik.version>1.10.0</morfologik.version>
+      <morfologik.version>2.0.1</morfologik.version>
       <languagetool.version>3.3-SNAPSHOT</languagetool.version>
   </properties>