From e33a362152441eb103d2c701878103993b74dac7 Mon Sep 17 00:00:00 2001 From: Francois Prunayre Date: Tue, 9 Jan 2024 12:41:12 +0100 Subject: [PATCH 1/3] Elasticsearch 8 migration. --- .../src/main/resources/application.yml | 2 +- modules/services/indexing/pom.xml | 8 +-- .../indexing/controller/IndexController.java | 2 +- .../indexing/service/IndexingService.java | 64 +++++++++---------- .../controller/ItemApiControllerTest.java | 4 ++ modules/services/searching/pom.xml | 5 -- pom.xml | 11 +--- 7 files changed, 43 insertions(+), 53 deletions(-) diff --git a/modules/library/common-search/src/main/resources/application.yml b/modules/library/common-search/src/main/resources/application.yml index c17007ae..84e1cb52 100644 --- a/modules/library/common-search/src/main/resources/application.yml +++ b/modules/library/common-search/src/main/resources/application.yml @@ -3,7 +3,7 @@ gn: # Full text on all fields # 'queryBase': '${any}', # Full text but more boost on title match - queryFilter: '+isTemplate:n AND -indexingError:true' + queryFilter: '+isTemplate:n' queryBase: '(any.\*:(${any}) OR resourceTitleObject.\*:(${any})^2)' trackTotalHits: true scoreConfig: > diff --git a/modules/services/indexing/pom.xml b/modules/services/indexing/pom.xml index 5ae9b157..494f5f7c 100644 --- a/modules/services/indexing/pom.xml +++ b/modules/services/indexing/pom.xml @@ -97,12 +97,8 @@ camel-micrometer-starter - org.elasticsearch.client - elasticsearch-rest-high-level-client - - - org.elasticsearch - elasticsearch + co.elastic.clients + elasticsearch-java diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java index 7bf2d3f7..5302066e 100644 --- a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java +++ b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java @@ -45,7 +45,7 @@ public ResponseEntity indexAll( @PostMapping("/event/{bucket}/{uuid}") public void sendEvent( @PathVariable("bucket")String bucket, - @PathVariable("uuid") String uuid) throws Exception { + @PathVariable("uuid") String uuid) { eventStreamService.produceEvent(bucket, uuid); } } diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java index ed0f4b29..5d3f501f 100644 --- a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java +++ b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java @@ -6,9 +6,14 @@ package org.fao.geonet.indexing.service; -import static org.elasticsearch.rest.RestStatus.CREATED; -import static org.elasticsearch.rest.RestStatus.OK; - +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.AcknowledgedResponse; +import co.elastic.clients.elasticsearch._types.Refresh; +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.IndexRequest; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -28,15 +33,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.xcontent.XContentType; import org.fao.geonet.common.xml.XsltUtil; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.Metadata; @@ -63,16 +60,19 @@ public class IndexingService { MetadataRepository metadataRepository; @Autowired - RestHighLevelClient client; + ElasticsearchClient client; /** * Delete index. */ public void deleteIndex(Exchange e) { try { + DeleteIndexRequest deleteIndexRequest = DeleteIndexRequest.of( + b -> b.index(index) + ); AcknowledgedResponse deleteIndexResponse = client.indices() - .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT); - if (deleteIndexResponse.isAcknowledged()) { + .delete(deleteIndexRequest); + if (deleteIndexResponse.acknowledged()) { log.info(String.format( "Index %s removed.", index)); @@ -202,20 +202,26 @@ protected static String collectDbProperties(AbstractMetadata r) { private BulkRequest buildBulkRequest(IndexRecords indexRecords) { - BulkRequest bulkRequest = new BulkRequest(index); + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .index(index) + .refresh(Refresh.True); ObjectMapper mapper = new ObjectMapper(); indexRecords.getIndexRecord().forEach(r -> { try { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.id(r.getId()); - indexRequest.source(mapper.writeValueAsString(r), XContentType.JSON); - bulkRequest.add(indexRequest); + String json = mapper.writeValueAsString(r); + requestBuilder.operations(op -> op + .index(idx -> idx + .index(index) + .id(r.getId()) + .document(json) + ) + ); } catch (JsonProcessingException jsonProcessingException) { jsonProcessingException.printStackTrace(); } }); - return bulkRequest; + return requestBuilder.build(); } private void sendToIndex(IndexRecords indexRecords, @@ -223,16 +229,16 @@ private void sendToIndex(IndexRecords indexRecords, BulkRequest bulkRequest = buildBulkRequest(indexRecords); try { // TODO: Asynchronous? - BulkResponse bulkItemResponses = client.bulk(bulkRequest, RequestOptions.DEFAULT); + BulkResponse bulkItemResponses = client.bulk(bulkRequest); log.info(String.format( "Indexing operation took %d.", - bulkItemResponses.getIngestTookInMillis() + bulkItemResponses.took() )); - if (bulkItemResponses.hasFailures()) { + if (bulkItemResponses.errors()) { AtomicInteger failureCount = new AtomicInteger(); - Arrays.stream(bulkItemResponses.getItems()).forEach(item -> { - if (item.status() != OK - && item.status() != CREATED) { + bulkItemResponses.items().forEach(item -> { + if (item.status() != 200 + && item.status() != 201) { failureCount.getAndIncrement(); // TODO: Index error document } @@ -243,12 +249,6 @@ private void sendToIndex(IndexRecords indexRecords, failureCount )); } - } catch (ElasticsearchStatusException indexException) { - report.setNumberOfRecordsWithIndexingErrors(indexRecords.getIndexRecord().size()); - log.error(String.format( - "Error while saving records %d in index. Error is: %s.", - indexException.getMessage() - )); } catch (IOException ioException) { log.error(String.format( "Error while sending records to index. Error is: %s.", diff --git a/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java b/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java index 435ba206..e15dc2b2 100644 --- a/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java +++ b/modules/services/ogc-api-records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java @@ -19,6 +19,7 @@ import org.fao.geonet.ogcapi.records.MvcConfigurer; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.MetadataRepository; +import org.fao.geonet.repository.SettingRepository; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.UiSettingsRepository; import org.fao.geonet.view.ViewUtility; @@ -64,6 +65,9 @@ public class ItemApiControllerTest { @MockBean private MetadataRepository mockMetadataRepository; + @MockBean + private SettingRepository mockSettingRepository; + @Autowired private MockMvc mockMvc; diff --git a/modules/services/searching/pom.xml b/modules/services/searching/pom.xml index a6ce956c..c89a0922 100644 --- a/modules/services/searching/pom.xml +++ b/modules/services/searching/pom.xml @@ -93,11 +93,6 @@ lombok - - org.springdoc springdoc-openapi-common diff --git a/pom.xml b/pom.xml index fbb064cc..67b3379b 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 4.2.8-0 9.4.27.v20200227 - 7.15.1 + 8.11.3 1.5.5 3.5.0 2.1.2 @@ -213,13 +213,8 @@ - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.elasticsearch - elasticsearch + co.elastic.clients + elasticsearch-java ${elasticsearch.version} From c0349b48797fba838afd780b115313c2a474c20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Prunayre?= Date: Wed, 28 Feb 2024 13:39:53 +0100 Subject: [PATCH 2/3] Update IndexingService.java --- .../org/fao/geonet/indexing/service/IndexingService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java index 5d3f501f..c73d1ac8 100644 --- a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java +++ b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java @@ -8,6 +8,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.AcknowledgedResponse; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; @@ -249,10 +250,10 @@ private void sendToIndex(IndexRecords indexRecords, failureCount )); } - } catch (IOException ioException) { + } catch (ElasticsearchException | IOException esException) { log.error(String.format( "Error while sending records to index. Error is: %s.", - ioException.getMessage() + esException.getMessage() )); } } From 31efe6fd0988100519af5393ecded8be35f0d5e3 Mon Sep 17 00:00:00 2001 From: Francois Prunayre Date: Fri, 1 Mar 2024 09:32:56 +0100 Subject: [PATCH 3/3] Versioning / Set dependency and version to 4.4.x branch. --- .env | 2 +- modules/library/common-error/pom.xml | 2 +- modules/library/common-index-model/pom.xml | 2 +- modules/library/common-persistence/pom.xml | 2 +- modules/library/common-search/pom.xml | 2 +- modules/library/common-security/pom.xml | 2 +- modules/library/common-standards/pom.xml | 2 +- modules/library/common-utility/pom.xml | 2 +- modules/library/common-view/pom.xml | 2 +- modules/library/pom.xml | 2 +- modules/pom.xml | 2 +- modules/services/authorizing/pom.xml | 2 +- modules/services/gateway/pom.xml | 2 +- modules/services/indexing/pom.xml | 2 +- modules/services/ogc-api-records/README.md | 2 +- modules/services/ogc-api-records/pom.xml | 2 +- modules/services/pom.xml | 2 +- modules/services/searching/pom.xml | 2 +- modules/standards/iso19139/pom.xml | 2 +- modules/standards/pom.xml | 2 +- modules/support-services/configuring/pom.xml | 2 +- modules/support-services/discovery/pom.xml | 2 +- modules/support-services/pom.xml | 2 +- pom.xml | 4 ++-- 24 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.env b/.env index 7240b77d..e7810b56 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ # docker-compose default environment variable values COMPOSE_PROJECT_NAME=gncloud -TAG=4.2.8-0 +TAG=4.4.3-0 EUREKA_SERVER_URL=http://discovery:8761/eureka # gn metadta postgres environment variables diff --git a/modules/library/common-error/pom.xml b/modules/library/common-error/pom.xml index e79a2943..59b41eca 100644 --- a/modules/library/common-error/pom.xml +++ b/modules/library/common-error/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/common-index-model/pom.xml b/modules/library/common-index-model/pom.xml index 12d09521..0ef1212d 100644 --- a/modules/library/common-index-model/pom.xml +++ b/modules/library/common-index-model/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/common-persistence/pom.xml b/modules/library/common-persistence/pom.xml index d059fa66..79174622 100644 --- a/modules/library/common-persistence/pom.xml +++ b/modules/library/common-persistence/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/common-search/pom.xml b/modules/library/common-search/pom.xml index c022dd2f..51764df5 100644 --- a/modules/library/common-search/pom.xml +++ b/modules/library/common-search/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/common-security/pom.xml b/modules/library/common-security/pom.xml index 4581d720..6278138c 100644 --- a/modules/library/common-security/pom.xml +++ b/modules/library/common-security/pom.xml @@ -11,7 +11,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-common-security diff --git a/modules/library/common-standards/pom.xml b/modules/library/common-standards/pom.xml index 8c7e7a26..a0f43fa3 100644 --- a/modules/library/common-standards/pom.xml +++ b/modules/library/common-standards/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/common-utility/pom.xml b/modules/library/common-utility/pom.xml index 64f1e051..0314e7a5 100644 --- a/modules/library/common-utility/pom.xml +++ b/modules/library/common-utility/pom.xml @@ -11,7 +11,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-common-utility diff --git a/modules/library/common-view/pom.xml b/modules/library/common-view/pom.xml index baeb868a..d051ce3c 100644 --- a/modules/library/common-view/pom.xml +++ b/modules/library/common-view/pom.xml @@ -3,7 +3,7 @@ gn-cloud-lib org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/library/pom.xml b/modules/library/pom.xml index 31d42eb9..f9a959c3 100644 --- a/modules/library/pom.xml +++ b/modules/library/pom.xml @@ -13,7 +13,7 @@ gn-cloud-parent org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 gn-cloud-lib pom diff --git a/modules/pom.xml b/modules/pom.xml index f24114b6..f29d1d88 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -13,7 +13,7 @@ gn-cloud-microservices org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 gn-cloud-parent pom diff --git a/modules/services/authorizing/pom.xml b/modules/services/authorizing/pom.xml index 5ee87d7c..6468a29f 100644 --- a/modules/services/authorizing/pom.xml +++ b/modules/services/authorizing/pom.xml @@ -10,7 +10,7 @@ gn-cloud-services org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 diff --git a/modules/services/gateway/pom.xml b/modules/services/gateway/pom.xml index 0f7b80c3..45fe6fdd 100644 --- a/modules/services/gateway/pom.xml +++ b/modules/services/gateway/pom.xml @@ -11,7 +11,7 @@ gn-cloud-services org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-gateway diff --git a/modules/services/indexing/pom.xml b/modules/services/indexing/pom.xml index 494f5f7c..461fd83a 100644 --- a/modules/services/indexing/pom.xml +++ b/modules/services/indexing/pom.xml @@ -11,7 +11,7 @@ gn-cloud-services org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-indexing diff --git a/modules/services/ogc-api-records/README.md b/modules/services/ogc-api-records/README.md index 3ad42e46..18128b9c 100644 --- a/modules/services/ogc-api-records/README.md +++ b/modules/services/ogc-api-records/README.md @@ -76,7 +76,7 @@ docker run -it -p8080:8080 \ -e "SPRING_PROFILES_ACTIVE=standalone" \ -e "SPRING_CONFIG_LOCATION=/ogcapiconfig/" \ -e "JAVA_OPTS=-Dfile.encoding=UTF-8" \ - gn-cloud-ogc-api-records-service:4.2.8-0 + gn-cloud-ogc-api-records-service:4.4.3-0 ``` or use a published release (create your configuration first like above): diff --git a/modules/services/ogc-api-records/pom.xml b/modules/services/ogc-api-records/pom.xml index 7cdfe42e..facb8348 100644 --- a/modules/services/ogc-api-records/pom.xml +++ b/modules/services/ogc-api-records/pom.xml @@ -10,7 +10,7 @@ org.geonetwork-opensource.cloud gn-cloud-services - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-ogc-api-records-service diff --git a/modules/services/pom.xml b/modules/services/pom.xml index b41caaca..1063231d 100644 --- a/modules/services/pom.xml +++ b/modules/services/pom.xml @@ -13,7 +13,7 @@ gn-cloud-parent org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 gn-cloud-services pom diff --git a/modules/services/searching/pom.xml b/modules/services/searching/pom.xml index c89a0922..dc9d8d08 100644 --- a/modules/services/searching/pom.xml +++ b/modules/services/searching/pom.xml @@ -11,7 +11,7 @@ org.geonetwork-opensource.cloud gn-cloud-services - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-searching diff --git a/modules/standards/iso19139/pom.xml b/modules/standards/iso19139/pom.xml index 3b4d7d35..234f5efa 100644 --- a/modules/standards/iso19139/pom.xml +++ b/modules/standards/iso19139/pom.xml @@ -3,7 +3,7 @@ gn-cloud-standards org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-standard-iso19139 diff --git a/modules/standards/pom.xml b/modules/standards/pom.xml index 88a61910..bb62ca50 100644 --- a/modules/standards/pom.xml +++ b/modules/standards/pom.xml @@ -4,7 +4,7 @@ gn-cloud-parent org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 gn-cloud-standards pom diff --git a/modules/support-services/configuring/pom.xml b/modules/support-services/configuring/pom.xml index a6b68317..1c19658b 100644 --- a/modules/support-services/configuring/pom.xml +++ b/modules/support-services/configuring/pom.xml @@ -10,7 +10,7 @@ gn-cloud-support-services org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-configuring diff --git a/modules/support-services/discovery/pom.xml b/modules/support-services/discovery/pom.xml index 978d9ce0..b21562b2 100644 --- a/modules/support-services/discovery/pom.xml +++ b/modules/support-services/discovery/pom.xml @@ -10,7 +10,7 @@ gn-cloud-support-services org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 4.0.0 gn-cloud-discovery diff --git a/modules/support-services/pom.xml b/modules/support-services/pom.xml index 57223534..a9daefd1 100644 --- a/modules/support-services/pom.xml +++ b/modules/support-services/pom.xml @@ -13,7 +13,7 @@ gn-cloud-parent org.geonetwork-opensource.cloud - 4.2.8-0 + 4.4.3-0 gn-cloud-support-services pom diff --git a/pom.xml b/pom.xml index 67b3379b..76af4692 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 4.0.0 org.geonetwork-opensource.cloud gn-cloud-microservices - 4.2.8-0 + 4.4.3-0 pom GeoNetwork microservices https://geonetwork-opensource.org @@ -28,7 +28,7 @@ Hoxton.SR8 2.3.8.RELEASE 5.3.2.RELEASE - 4.2.8-0 + 4.4.2-0 9.4.27.v20200227 8.11.3