SOLR-17575: Fixed broken backwards compatibility with the legacy "lan…

…gid.whitelist" config in Solr Langid (#2886) Co-authored-by: Alexander Zagniotov <[email protected]> Co-authored-by: Jan Høydahl <[email protected]>
apache · Nov 28, 2024 · cebdb2d · cebdb2d
1 parent 64130ec
commit cebdb2d
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 7 deletions.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -227,6 +227,8 @@ Bug Fixes
 * SOLR-16976: Remove log4j-jul jar and use slf4j bridge for JUL to prevent exception from being logged when remote JMX
   is enabled (Shawn Heisey, Stephen Zhou, Eric Pugh, Christine Poerschke, David Smiley)
 
+* SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid. (Jan Høydahl, Alexander Zagniotov)
+
 Dependency Upgrades
 ---------------------
 (No changes)

diff --git a/...s/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/...s/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
@@ -111,20 +111,19 @@ private void initParams(SolrParams params) {
       overwrite = params.getBool(OVERWRITE, false);
       langAllowlist = new HashSet<>();
       threshold = params.getDouble(THRESHOLD, DOCID_THRESHOLD_DEFAULT);
-      String legacyAllowList = params.get(LANG_WHITELIST, "");
-      if (legacyAllowList.length() > 0) {
+      final String legacyAllowList = params.get(LANG_WHITELIST, "").trim();
+      if (!legacyAllowList.isEmpty()) {
         // nowarn compile time string concatenation
         log.warn(
             LANG_WHITELIST
                 + " parameter is deprecated; use "
                 + LANG_ALLOWLIST
                 + " instead."); // nowarn
       }
-      if (params.get(LANG_ALLOWLIST, legacyAllowList).length() > 0) {
-        for (String lang : params.get(LANG_ALLOWLIST, "").split(",")) {
-          langAllowlist.add(lang);
-        }
-      }
+      Arrays.stream(params.get(LANG_ALLOWLIST, legacyAllowList).split(","))
+          .map(String::trim)
+          .filter(lang -> !lang.isEmpty())
+          .forEach(langAllowlist::add);
 
       // Mapping params (field centric)
       enableMapping = params.getBool(MAP_ENABLE, false);

diff --git a/...st/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java b/...st/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
@@ -18,6 +18,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -464,6 +465,31 @@ public void testMapIndividual() throws Exception {
     assertTrue(mappedIndividual.containsKey("text2_ru"));
   }
 
+  @Test
+  public void testAllowlist() throws Exception {
+    ModifiableSolrParams parameters = new ModifiableSolrParams();
+    parameters.add("langid.fl", "name,subject");
+    parameters.add("langid.langField", "language_s");
+    parameters.add("langid.allowlist", "no,en ,, ,sv, sv");
+    liProcessor = createLangIdProcessor(parameters);
+
+    // Make sure that empty language codes have been filtered out and others trimmed.
+    assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);
+  }
+
+  @Test
+  public void testAllowlistBackwardsCompatabilityWithLegacyAllowlist() throws Exception {
+    // The "legacy allowlist" is "langid.whitelist"
+    ModifiableSolrParams parameters = new ModifiableSolrParams();
+    parameters.add("langid.fl", "name,subject");
+    parameters.add("langid.langField", "language_s");
+    parameters.add("langid.whitelist", "no,en ,, ,sv, sv");
+    liProcessor = createLangIdProcessor(parameters);
+
+    // Make sure that empty language codes have been filtered out and others trimmed.
+    assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);
+  }
+
   // Various utility methods
 
   private SolrInputDocument englishDoc() {