Elasticsearch 8 migration.

geonetwork · Jan 9, 2024 · d96967d · d96967d
1 parent 5bb8909
commit d96967d
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 53 deletions.
diff --git a/modules/library/common-search/src/main/resources/application.yml b/modules/library/common-search/src/main/resources/application.yml
@@ -3,7 +3,7 @@ gn:
     # Full text on all fields
     # 'queryBase': '${any}',
     # Full text but more boost on title match
-    queryFilter: '+isTemplate:n AND -indexingError:true'
+    queryFilter: '+isTemplate:n'
     queryBase: '(any.\*:(${any}) OR resourceTitleObject.\*:(${any})^2)'
     trackTotalHits: true
     scoreConfig: >

diff --git a/modules/services/indexing/pom.xml b/modules/services/indexing/pom.xml
@@ -97,12 +97,8 @@
       <artifactId>camel-micrometer-starter</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.elasticsearch</groupId>
-      <artifactId>elasticsearch</artifactId>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
     </dependency>
 
     <!--  test dependencies  -->

diff --git a/...s/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java b/...s/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java
@@ -45,7 +45,7 @@ public ResponseEntity<?> indexAll(
   @PostMapping("/event/{bucket}/{uuid}")
   public void sendEvent(
       @PathVariable("bucket")String bucket, 
-      @PathVariable("uuid") String uuid) throws Exception {
+      @PathVariable("uuid") String uuid) {
     eventStreamService.produceEvent(bucket, uuid);
   }
 }
diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java
@@ -6,9 +6,14 @@
 
 package org.fao.geonet.indexing.service;
 
-import static org.elasticsearch.rest.RestStatus.CREATED;
-import static org.elasticsearch.rest.RestStatus.OK;
-
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.AcknowledgedResponse;
+import co.elastic.clients.elasticsearch._types.Refresh;
+import co.elastic.clients.elasticsearch.core.BulkRequest;
+import co.elastic.clients.elasticsearch.core.BulkResponse;
+import co.elastic.clients.elasticsearch.core.IndexRequest;
+import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
+import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.File;
@@ -28,15 +33,7 @@
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.camel.Exchange;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.fao.geonet.common.xml.XsltUtil;
 import org.fao.geonet.domain.AbstractMetadata;
 import org.fao.geonet.domain.Metadata;
@@ -63,16 +60,19 @@ public class IndexingService {
   MetadataRepository metadataRepository;
 
   @Autowired
-  RestHighLevelClient client;
+  ElasticsearchClient client;
 
   /**
    * Delete index.
    */
   public void deleteIndex(Exchange e) {
     try {
+      DeleteIndexRequest deleteIndexRequest = DeleteIndexRequest.of(
+          b -> b.index(index)
+      );
       AcknowledgedResponse deleteIndexResponse = client.indices()
-          .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT);
-      if (deleteIndexResponse.isAcknowledged()) {
+          .delete(deleteIndexRequest);
+      if (deleteIndexResponse.acknowledged()) {
         log.info(String.format(
             "Index %s removed.",
             index));
@@ -202,37 +202,43 @@ protected static String collectDbProperties(AbstractMetadata r) {
 
 
   private BulkRequest buildBulkRequest(IndexRecords indexRecords) {
-    BulkRequest bulkRequest = new BulkRequest(index);
+    BulkRequest.Builder requestBuilder = new BulkRequest.Builder()
+        .index(index)
+        .refresh(Refresh.True);
     ObjectMapper mapper = new ObjectMapper();
 
     indexRecords.getIndexRecord().forEach(r -> {
       try {
-        IndexRequest indexRequest = new IndexRequest();
-        indexRequest.id(r.getId());
-        indexRequest.source(mapper.writeValueAsString(r), XContentType.JSON);
-        bulkRequest.add(indexRequest);
+        String json = mapper.writeValueAsString(r);
+        requestBuilder.operations(op -> op
+            .index(idx -> idx
+                .index(index)
+                .id(r.getId())
+                .document(json)
+            )
+        );
       } catch (JsonProcessingException jsonProcessingException) {
         jsonProcessingException.printStackTrace();
       }
     });
-    return bulkRequest;
+    return requestBuilder.build();
   }
 
   private void sendToIndex(IndexRecords indexRecords,
       IndexingReport report) {
     BulkRequest bulkRequest = buildBulkRequest(indexRecords);
     try {
       // TODO: Asynchronous?
-      BulkResponse bulkItemResponses = client.bulk(bulkRequest, RequestOptions.DEFAULT);
+      BulkResponse bulkItemResponses = client.bulk(bulkRequest);
       log.info(String.format(
           "Indexing operation took %d.",
-          bulkItemResponses.getIngestTookInMillis()
+          bulkItemResponses.took()
       ));
-      if (bulkItemResponses.hasFailures()) {
+      if (bulkItemResponses.errors()) {
         AtomicInteger failureCount = new AtomicInteger();
-        Arrays.stream(bulkItemResponses.getItems()).forEach(item -> {
-          if (item.status() != OK
-              && item.status() != CREATED) {
+        bulkItemResponses.items().forEach(item -> {
+          if (item.status() != 200
+              && item.status() != 201) {
             failureCount.getAndIncrement();
             // TODO: Index error document
           }
@@ -243,12 +249,6 @@ private void sendToIndex(IndexRecords indexRecords,
             failureCount
         ));
       }
-    } catch (ElasticsearchStatusException indexException) {
-      report.setNumberOfRecordsWithIndexingErrors(indexRecords.getIndexRecord().size());
-      log.error(String.format(
-          "Error while saving records %d in index. Error is: %s.",
-          indexException.getMessage()
-      ));
     } catch (IOException ioException) {
       log.error(String.format(
           "Error while sending records to index. Error is: %s.",

diff --git a/...records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java b/...records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java
@@ -19,6 +19,7 @@
 import org.fao.geonet.ogcapi.records.MvcConfigurer;
 import org.fao.geonet.repository.LanguageRepository;
 import org.fao.geonet.repository.MetadataRepository;
+import org.fao.geonet.repository.SettingRepository;
 import org.fao.geonet.repository.SourceRepository;
 import org.fao.geonet.repository.UiSettingsRepository;
 import org.fao.geonet.view.ViewUtility;
@@ -64,6 +65,9 @@ public class ItemApiControllerTest {
     @MockBean
     private MetadataRepository mockMetadataRepository;
 
+    @MockBean
+    private SettingRepository mockSettingRepository;
+
     @Autowired
     private MockMvc mockMvc;
 

diff --git a/modules/services/searching/pom.xml b/modules/services/searching/pom.xml
@@ -93,11 +93,6 @@
       <artifactId>lombok</artifactId>
     </dependency>
 
-    <!--<dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-    </dependency>-->
-
     <dependency>
       <groupId>org.springdoc</groupId>
       <artifactId>springdoc-openapi-common</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -31,7 +31,7 @@
     <geonetwork.version>4.2.7-0</geonetwork.version>
     <jetty.version>9.4.27.v20200227</jetty.version>
 
-    <elasticsearch.version>7.15.1</elasticsearch.version>
+    <elasticsearch.version>8.11.3</elasticsearch.version>
     <micrometer.version>1.5.5</micrometer.version>
     <camel.version>3.5.0</camel.version>
     <swagger.version>2.1.2</swagger.version>
@@ -213,13 +213,8 @@
         </exclusions>
       </dependency>
       <dependency>
-        <groupId>org.elasticsearch.client</groupId>
-        <artifactId>elasticsearch-rest-high-level-client</artifactId>
-        <version>${elasticsearch.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.elasticsearch</groupId>
-        <artifactId>elasticsearch</artifactId>
+        <groupId>co.elastic.clients</groupId>
+        <artifactId>elasticsearch-java</artifactId>
         <version>${elasticsearch.version}</version>
       </dependency>
       <dependency>