Upgrade to Solr 9.0

- Updated Solr schema to the latest version. - Removed deprecated HttpSolrClientFactory and SolrClientProxy. - Removed Solr core and Lucene dependencies since the embedded Solr server was removed in a previous release. - Embedded JTS and Spatial4j in Solr related bundles since the version of JTS required for Spatial4j is not compatible with Geotools and JTS does not follow semantic versioning. - Updated the Failsafe retry library to help simplify the Solr client collection creation and provider field cache retry logic.
codice · Jun 28, 2022 · 4dce087 · 4dce087
1 parent b038d85
commit 4dce087
Show file tree

Hide file tree

Showing 129 changed files with 1,085 additions and 9,796 deletions.
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
@@ -0,0 +1,10 @@
+--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
diff --git a/catalog/catalog-app/src/main/resources/features.xml b/catalog/catalog-app/src/main/resources/features.xml
@@ -62,7 +62,7 @@
         </bundle>
         <bundle>mvn:org.apache.geronimo.specs/geronimo-jms_1.1_spec/1.1.1</bundle>
         <bundle>mvn:org.codice.thirdparty/lucene-core/3.0.2_1</bundle>
-        <feature>jodah-failsafe</feature>
+        <feature>dev-failsafe</feature>
         <bundle>mvn:ddf.catalog.core/ddf-pubsub/${project.version}</bundle>
         <bundle>mvn:ddf.catalog.core/ddf-pubsub-tracker/${project.version}</bundle>
         <bundle>mvn:ddf.catalog.core/catalog-core-urlresourcereader/${project.version}</bundle>
@@ -73,10 +73,10 @@
         <bundle>mvn:ddf.catalog.core/catalog-core-localstorageprovider/${project.version}</bundle>
         <bundle>mvn:ddf.catalog.core/versioning-api/${project.version}</bundle>
         <feature>joda-converter</feature>
-        <feature>jodah-failsafe</feature>
+        <feature>dev-failsafe</feature>
         <feature>slf4j</feature>
         <bundle>mvn:org.locationtech.spatial4j/spatial4j/${spatial4j.version}</bundle>
-        <bundle>mvn:net.jodah/failsafe/${jodah-failsafe.version}</bundle>
+        <bundle>mvn:dev.failsafe/failsafe/${dev.failsafe.version}</bundle>
 
         <bundle>mvn:nu.validator.htmlparser/htmlparser/1.4</bundle>
         <bundle>mvn:ddf.catalog.core/catalog-core-standardframework/${project.version}</bundle>
@@ -305,7 +305,7 @@
         <feature>apache-commons</feature>
         <feature>catalog-transformer-bootflag</feature>
         <bundle>mvn:com.hazelcast/hazelcast/${hazelcast.version}</bundle>
-        <bundle>mvn:net.jodah/failsafe/${jodah-failsafe.version}</bundle>
+        <bundle>mvn:dev.failsafe/failsafe/${dev.failsafe.version}</bundle>
         <bundle>mvn:com.google.code.gson/gson/${gson.version}</bundle>
         <bundle>mvn:ddf.catalog.core/catalog-core-directorymonitor/${project.version}</bundle>
         <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.javax-cache-api/${javax.cache.bundle.version}</bundle>
@@ -316,7 +316,7 @@
         <feature>catalog-core-api</feature>
         <feature>bootflag-api</feature>
         <bundle>mvn:com.google.code.gson/gson/${gson.version}</bundle>
-        <bundle>mvn:net.jodah/failsafe/${jodah-failsafe.version}</bundle>
+        <bundle>mvn:dev.failsafe/failsafe/${dev.failsafe.version}</bundle>
         <bundle>mvn:ddf.catalog.transformer/catalog-transformer-bootflag/${project.version}</bundle>
     </feature>
 
@@ -421,7 +421,7 @@
 
     <feature name="catalog-transformer-atom" version="${project.version}"
              description="Atom Query Response Transformer.">
-        <feature>jodah-failsafe</feature>
+        <feature>dev-failsafe</feature>
         <bundle>mvn:org.apache.abdera/abdera-extensions-opensearch/${abdera.version}</bundle>
         <bundle>mvn:org.apache.abdera/abdera-extensions-geo/${abdera.version}</bundle>
         <bundle>mvn:org.apache.abdera/abdera-server/${abdera.version}</bundle>

diff --git a/...nt/src/main/java/ddf/camel/component/catalog/content/ContentProducerDataAccessObject.java b/...nt/src/main/java/ddf/camel/component/catalog/content/ContentProducerDataAccessObject.java
@@ -35,6 +35,8 @@
 import ddf.catalog.source.SourceUnavailableException;
 import ddf.mime.MimeTypeMapper;
 import ddf.mime.MimeTypeResolutionException;
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -43,17 +45,14 @@
 import java.nio.file.StandardWatchEventKinds;
 import java.nio.file.WatchEvent;
 import java.nio.file.WatchEvent.Kind;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import net.jodah.failsafe.Failsafe;
-import net.jodah.failsafe.RetryPolicy;
-import net.jodah.failsafe.function.Predicate;
 import org.apache.camel.Message;
 import org.apache.camel.component.file.GenericFile;
 import org.apache.camel.component.file.GenericFileMessage;
@@ -255,13 +254,14 @@ protected HashMap<String, Serializable> getProperties(Map<String, Object> header
 
   private void waitForAvailableSource(CatalogFramework catalogFramework)
       throws SourceUnavailableException {
-    RetryPolicy retryPolicy =
-        new RetryPolicy()
-            .withDelay(3, TimeUnit.SECONDS)
-            .withMaxDuration(3, TimeUnit.MINUTES)
-            .retryIf((Predicate<Set>) Set::isEmpty)
-            .retryIf(
-                (Set<SourceDescriptor> result) -> !result.stream().findFirst().get().isAvailable());
+    RetryPolicy<Set<SourceDescriptor>> retryPolicy =
+        RetryPolicy.<Set<SourceDescriptor>>builder()
+            .withDelay(Duration.ofSeconds(3))
+            .withMaxDuration(Duration.ofMinutes(3))
+            .withMaxRetries(-1)
+            .handleResultIf(Set::isEmpty)
+            .handleResultIf(result -> !result.stream().findFirst().get().isAvailable())
+            .build();
 
     Failsafe.with(retryPolicy)
         .get(

diff --git a/catalog/core/catalog-core-directorymonitor/pom.xml b/catalog/core/catalog-core-directorymonitor/pom.xml
@@ -40,9 +40,9 @@
             <artifactId>joda-time</artifactId>
         </dependency>
         <dependency>
-            <groupId>net.jodah</groupId>
+            <groupId>dev.failsafe</groupId>
             <artifactId>failsafe</artifactId>
-            <version>${jodah-failsafe.version}</version>
+            <version>${dev.failsafe.version}</version>
         </dependency>
         <dependency>
             <groupId>org.osgi</groupId>

diff --git a/...monitor/src/main/java/org/codice/ddf/catalog/content/monitor/ContentDirectoryMonitor.java b/...monitor/src/main/java/org/codice/ddf/catalog/content/monitor/ContentDirectoryMonitor.java
@@ -17,7 +17,10 @@
 
 import ddf.catalog.Constants;
 import ddf.catalog.data.AttributeRegistry;
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
 import java.io.Serializable;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -26,12 +29,10 @@
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
-import net.jodah.failsafe.Failsafe;
-import net.jodah.failsafe.RetryPolicy;
 import org.apache.camel.CamelContext;
+import org.apache.camel.Component;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.apache.camel.ServiceStatus;
@@ -328,16 +329,17 @@ private void attemptAddRoutes() {
   }
 
   private void verifyCamelComponentIsAvailable(String componentName) {
-    Failsafe.with(
-            new RetryPolicy()
-                .retryWhen(null)
-                .withMaxRetries(maxRetries)
-                .withDelay(delayBetweenRetries, TimeUnit.SECONDS))
-        .withFallback(
-            () -> {
-              throw new IllegalStateException("Could not get Camel component " + componentName);
-            })
-        .get(() -> camelContext.getComponent(componentName));
+    Component component =
+        Failsafe.with(
+                RetryPolicy.<Component>builder()
+                    .handleResult(null)
+                    .withMaxRetries(maxRetries)
+                    .withDelay(Duration.ofSeconds(delayBetweenRetries))
+                    .build())
+            .get(() -> camelContext.getComponent(componentName));
+    if (component == null) {
+      throw new IllegalStateException("Could not get Camel component " + componentName);
+    }
   }
 
   private RouteBuilder createRouteBuilder() {

diff --git a/...tor/src/test/java/org/codice/ddf/catalog/content/monitor/ContentDirectoryMonitorTest.java b/...tor/src/test/java/org/codice/ddf/catalog/content/monitor/ContentDirectoryMonitorTest.java
@@ -22,14 +22,15 @@
 import static org.mockito.Mockito.mock;
 
 import ddf.catalog.data.AttributeRegistry;
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
 import java.io.File;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import net.jodah.failsafe.Failsafe;
-import net.jodah.failsafe.RetryPolicy;
 import org.apache.camel.CamelContext;
 import org.apache.camel.Exchange;
 import org.apache.camel.component.mock.MockComponent;
@@ -305,14 +306,11 @@ private void doAndVerifyFileMove(
       File destinationFolder, File monitoredFolder, String inputFileName) throws Exception {
     doFileMove(destinationFolder, inputFileName);
     Failsafe.with(
-            new RetryPolicy()
-                .retryWhen(false)
+            RetryPolicy.<Boolean>builder()
+                .handleResult(false)
                 .withMaxRetries(MAX_CHECKS_FOR_FILE_COPY)
-                .withDelay(5, TimeUnit.SECONDS))
-        .withFallback(
-            () -> {
-              throw new RuntimeException("File did not get moved in time");
-            })
+                .withDelay(Duration.ofSeconds(5))
+                .build())
         .get(() -> verifyFileMovedToIngestedDirectory(monitoredFolder, inputFileName));
 
     assertThat(

diff --git a/catalog/core/catalog-core-standardframework/pom.xml b/catalog/core/catalog-core-standardframework/pom.xml
@@ -126,9 +126,9 @@
             <artifactId>guava</artifactId>
         </dependency>
         <dependency>
-            <groupId>net.jodah</groupId>
+            <groupId>dev.failsafe</groupId>
             <artifactId>failsafe</artifactId>
-            <version>0.9.3</version>
+            <version>${dev.failsafe.version}</version>
         </dependency>
         <dependency>
             <groupId>jakarta.xml.bind</groupId>

diff --git a/catalog/security/catalog-security-resourceplugin/pom.xml b/catalog/security/catalog-security-resourceplugin/pom.xml
@@ -45,9 +45,9 @@
             <artifactId>commons-lang3</artifactId>
         </dependency>
         <dependency>
-            <groupId>net.jodah</groupId>
+            <groupId>dev.failsafe</groupId>
             <artifactId>failsafe</artifactId>
-            <version>1.0.0</version>
+            <version>${dev.failsafe.version}</version>
         </dependency>
         <dependency>
             <groupId>ddf.security.core</groupId>

diff --git a/catalog/solr/catalog-solr-app/pom.xml b/catalog/solr/catalog-solr-app/pom.xml
@@ -72,11 +72,6 @@
         </plugins>
     </build>
     <dependencies>
-        <dependency>
-            <groupId>org.codice.thirdparty</groupId>
-            <artifactId>jts</artifactId>
-            <version>${jts.bundle.version}</version>
-        </dependency>
         <dependency>
             <groupId>ddf.catalog.solr</groupId>
             <artifactId>catalog-solr-cache</artifactId>

diff --git a/catalog/solr/catalog-solr-app/src/main/resources/features.xml b/catalog/solr/catalog-solr-app/src/main/resources/features.xml
@@ -60,7 +60,6 @@
              description="The Solr Catalog Provider (SCP) is an implementation of the CatalogProvider interface using Apache Solr ${solr.version} as a data store.\nThe SCP supports extensible metacards, fast/simple contextual searching, indexes xml attributes/CDATA sections/XML text elements, contains full XPath support, works with an embedded local Solr Server, and also works with a standalone Solr Server.">
         <feature>solr-factory-api</feature>
         <feature>solr-factory-impl</feature>
-        <bundle>mvn:org.codice.thirdparty/jts/${jts.bundle.version}</bundle>
         <feature>catalog-solr-provider</feature>
         <feature>catalog-solr-solrclient</feature>
         <feature>catalog-solr-commands</feature>

diff --git a/catalog/solr/catalog-solr-cache/pom.xml b/catalog/solr/catalog-solr-cache/pom.xml
@@ -47,6 +47,16 @@
             <artifactId>catalog-solr-core</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.locationtech.jts</groupId>
+            <artifactId>jts-core</artifactId>
+            <version>${jts.spatial4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.locationtech.spatial4j</groupId>
+            <artifactId>spatial4j</artifactId>
+            <version>${spatial4j.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -73,6 +83,8 @@
                     <instructions>
                         <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
                         <Embed-Dependency>
+                            jts-core,
+                            spatial4j,
                             catalog-solr-core,
                             catalog-core-api-impl
                         </Embed-Dependency>
@@ -85,28 +97,15 @@
                         </Export-Package>
                         <Import-Package>
                             <!--packages required for solrj -->
+                            !org.noggit*,
                             org.apache.jute;version=${solr.zookeeper.version},
                             org.apache.solr.client.solrj;version=${solr.version},
-                            org.apache.solr.client.solrj.beans;version=${solr.version},
-                            org.apache.solr.client.solrj.cloud;version=${solr.version},
-                            org.apache.solr.client.solrj.cloud.autoscaling;version=${solr.version},
-                            org.apache.solr.client.solrj.impl;version=${solr.version},
-                            org.apache.solr.client.solrj.request;version=${solr.version},
-                            org.apache.solr.client.solrj.response;version=${solr.version},
+                            org.apache.solr.client.solrj.*;version=${solr.version},
                             org.apache.solr.common;version=${solr.version},
-                            org.apache.solr.common.params;version=${solr.version},
-                            org.apache.solr.common.util;version=${solr.version},
+                            org.apache.solr.common.*;version=${solr.version},
                             org.apache.zookeeper;version=${solr.zookeeper.version},
-                            org.apache.zookeeper.client;version=${solr.zookeeper.version},
-                            org.apache.zookeeper.data;version=${solr.zookeeper.version},
-                            org.apache.zookeeper.proto;version=${solr.zookeeper.version},
+                            org.apache.zookeeper.*;version=${solr.zookeeper.version},
                             com.fasterxml.jackson.*;version=${jackson.version},
-
-                            <!-- dependencies used by catalog-solr-core-->
-                            org.locationtech.spatial4j.*;version=${solr.spatial4j.version},
-                            org.locationtech.jts.*;version=${jts.spatial4j.version},
-                            com.saxonica.config;resolution:=optional,
-                            org.expath.pkg.saxon;resolution:=optional,
                             *
                         </Import-Package>
                     </instructions>

diff --git a/...r-cache/src/main/java/org/codice/ddf/catalog/solr/cache/impl/CacheSolrMetacardClient.java b/...r-cache/src/main/java/org/codice/ddf/catalog/solr/cache/impl/CacheSolrMetacardClient.java
@@ -25,11 +25,11 @@
 import java.util.Date;
 import java.util.List;
 import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrInputDocument;
-import org.codice.solr.client.solrj.SolrClient;
 
 /** {@link SolrCache} specific implementation of {@link SolrMetacardClientImpl}. */
 class CacheSolrMetacardClient extends SolrMetacardClientImpl {

diff --git a/...lr/catalog-solr-cache/src/main/java/org/codice/ddf/catalog/solr/cache/impl/SolrCache.java b/...lr/catalog-solr-cache/src/main/java/org/codice/ddf/catalog/solr/cache/impl/SolrCache.java
@@ -41,12 +41,11 @@
 import java.util.function.Supplier;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.common.SolrException;
 import org.codice.ddf.catalog.solr.cache.CachePutPlugin;
 import org.codice.ddf.platform.util.StandardThreadFactoryBuilder;
-import org.codice.solr.client.solrj.SolrClient;
-import org.codice.solr.client.solrj.UnavailableSolrException;
 import org.codice.solr.factory.SolrClientFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -302,8 +301,6 @@ public void run() {
       try {
         LOGGER.debug("Expiring cache.");
         client.deleteByQuery(CACHED_DATE + ":[* TO NOW-" + expirationAgeInMinutes + "MINUTES]");
-      } catch (UnavailableSolrException e) {
-        LOGGER.debug("Unable to expire cache.", e);
       } catch (SolrServerException | SolrException | IOException e) {
         LOGGER.info("Unable to expire cache; {}", e.getMessage());
         LOGGER.debug("Cache expiration error.", e);

diff --git a/...atalog-solr-cache/src/test/java/org/codice/ddf/catalog/solr/cache/impl/SolrCacheTest.java b/...atalog-solr-cache/src/test/java/org/codice/ddf/catalog/solr/cache/impl/SolrCacheTest.java
@@ -39,9 +39,9 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ScheduledExecutorService;
+import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.codice.ddf.catalog.solr.cache.CachePutPlugin;
-import org.codice.solr.client.solrj.SolrClient;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;