added keyClasses to searchableText, added sorting to search endpoint (#…

…93) * added params to search API * rights holder is now indexed * rights holder is now indexed * rights holder is now indexed * fixes owasp * using a different index name to avoid reindexing * added keyClasses to searchableText, added sorting to search endpoint * added keyClasses to searchableText, added sorting to search endpoint * added keyClasses to searchableText, added sorting to search endpoint * fixes some tests
teamdigitale · Mar 7, 2024 · 05ae298 · 05ae298
1 parent da3ec4d
commit 05ae298
Show file tree

Hide file tree

Showing 11 changed files with 104 additions and 11 deletions.
diff --git a/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java b/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java
@@ -7,10 +7,11 @@
 import it.gov.innovazione.ndc.model.Builders;
 import it.gov.innovazione.ndc.service.SemanticAssetSearchService;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mock;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
 import org.springframework.http.MediaType;
 import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.web.servlet.ResultActions;
@@ -76,12 +77,15 @@ void shouldReturnMatchingAssetsUsingDefaultPageParams() throws Exception {
         dto.setDescription("some-description");
         dto.setModifiedOn(LocalDate.parse("2020-01-01"));
 
+        PageRequest pageable = OffsetBasedPageRequest.of(0, 10, Sort.by(Sort.Order.asc("title")));
+
         when(searchService.search(any(), any(), any(), any(), any())
         ).thenReturn(Builders.searchResult()
                 .limit(10)
                 .offset(0)
                 .totalCount(1)
                 .data(List.of(dto))
+                .pageable(pageable)
                 .build());
 
         ResultActions apiResult = mockMvc.perform(get("/semantic-assets")
@@ -90,6 +94,8 @@ void shouldReturnMatchingAssetsUsingDefaultPageParams() throws Exception {
                 .param("type", "ONTOLOGY")
                 .param("theme", "http://publications.europa.eu/resource/authority/data-theme/AGRI")
                 .param("theme", "http://publications.europa.eu/resource/authority/data-theme/EDUC")
+                .param("sortBy", "TITLE")
+                .param("direction", "ASC")
                 .accept(MediaType.APPLICATION_JSON)
         );
 
@@ -108,18 +114,20 @@ void shouldReturnMatchingAssetsUsingDefaultPageParams() throws Exception {
                 Set.of("CONTROLLED_VOCABULARY", "ONTOLOGY"),
                 Set.of("http://publications.europa.eu/resource/authority/data-theme/AGRI", "http://publications.europa.eu/resource/authority/data-theme/EDUC"),
                 null,
-                OffsetBasedPageRequest.of(0, 10));
+                pageable);
     }
 
     @Test
     void shouldReturnMatchingAssetsUsingProvidedPageParams() throws Exception {
         SearchResultItem dto = new SearchResultItem();
+        PageRequest pageable = OffsetBasedPageRequest.of(100, 20, Sort.by(Sort.Order.asc("title")));
         when(searchService.search(any(), any(), any(), any(), any())
         ).thenReturn(Builders.searchResult()
                 .limit(20)
                 .offset(100)
                 .totalCount(101)
                 .data(List.of(dto))
+                .pageable(pageable)
                 .build());
 
         ResultActions apiResult = mockMvc.perform(get("/semantic-assets")
@@ -128,7 +136,7 @@ void shouldReturnMatchingAssetsUsingProvidedPageParams() throws Exception {
                 .accept(MediaType.APPLICATION_JSON)
         );
 
-        verify(searchService).search("", Set.of(), Set.of(), null, OffsetBasedPageRequest.of(100, 20));
+        verify(searchService).search("", Set.of(), Set.of(), null, pageable);
 
         apiResult
                 .andDo(print())
@@ -145,7 +153,8 @@ void shouldSearchWithDefaultWhenNoParamsProvided() throws Exception {
                 .andDo(print())
                 .andExpect(status().isOk());
 
-        verify(searchService).search("", Set.of(), Set.of(), null, OffsetBasedPageRequest.of(0, 10));
+        verify(searchService).search("", Set.of(), Set.of(), null, OffsetBasedPageRequest.of(0, 10,
+                Sort.by(Sort.Order.asc("title"))));
     }
 
     @Test

diff --git a/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java b/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java
@@ -1,5 +1,6 @@
 package it.gov.innovazione.ndc.controller;
 
+import com.github.jsonldjava.shaded.com.google.common.base.CaseFormat;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiResponse;
 import io.swagger.annotations.ApiResponses;
@@ -15,6 +16,7 @@
 import it.gov.innovazione.ndc.service.SemanticAssetSearchService;
 import lombok.RequiredArgsConstructor;
 import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Sort;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -27,6 +29,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static it.gov.innovazione.ndc.gen.dto.Direction.ASC;
+
 @RequiredArgsConstructor
 @RestController
 public class SemanticAssetsController implements SemanticAssetsApi {
@@ -63,7 +67,12 @@ public ResponseEntity<SearchResult> search(
             Set<Theme> theme,
             Set<String> rightsHolder) {
 
-        Pageable pageable = OffsetBasedPageRequest.of(offset, limit);
+        String property = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, sortBy.getValue());
+
+        Pageable pageable = OffsetBasedPageRequest.of(offset, limit,
+                Sort.by(direction == ASC
+                        ? Sort.Order.asc(property)
+                        : Sort.Order.desc(property)));
 
         return AppJsonResponse.ok(
                 searchService.search(q,

diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java
@@ -23,6 +23,7 @@
 import javax.xml.bind.DatatypeConverter;
 import java.time.LocalDate;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Consumer;
@@ -225,6 +226,7 @@ public SemanticAssetMetadata extractMetadata() {
                 .distributions(getDistributions())
                 .status(LiteralExtractor.extractAll(mainResource, Admsapit.status))
                 .agencyId(agencyId.getIdentifier())
+                .agencyLabel(new ArrayList<>(agencyId.getName().values()))
                 .build();
     }
 

diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java
@@ -15,6 +15,7 @@
 import org.apache.jena.sparql.vocabulary.FOAF;
 import org.apache.jena.vocabulary.RDFS;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -77,6 +78,7 @@ public SemanticAssetMetadata extractMetadata() {
                 .keyClasses(getKeyClasses())
                 .status(LiteralExtractor.extractAll(mainResource, Admsapit.status))
                 .agencyId(rightsHolderObj.getIdentifier())
+                .agencyLabel(new ArrayList<>(rightsHolderObj.getName().values()))
                 .build();
     }
 

diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java
@@ -32,8 +32,12 @@ public class SemanticAssetMetadata {
     @Field(type = Keyword)
     private String repoUrl;
 
-    @Field(type = Text, copyTo = "searchableText")
+
+    @Field(index = false, type = Keyword, normalizer = "lowercase_normalizer")
     private String title;
+    @Field(type = Text, copyTo = "searchableText")
+    private String rawTitle;
+
     @Field(type = Text, copyTo = "searchableText")
     private String description;
     @Field(type = Keyword, copyTo = "searchableText")
@@ -76,6 +80,11 @@ public class SemanticAssetMetadata {
     private String keyConcept;
     @Field(type = Keyword)
     private String agencyId;
+
+    @JsonIgnore
+    @Field(type = Text, copyTo = "searchableText")
+    private List<String> agencyLabel;
+
     @Field(index = false, type = Keyword)
     private String endpointUrl;
 
@@ -89,6 +98,9 @@ public class SemanticAssetMetadata {
     @Field(index = false, type = FieldType.Object)
     private List<NodeSummary> keyClasses;
 
+    @Field(type = Text, copyTo = "searchableText")
+    private List<String> keyClassesLabels;
+
     //for the searchable content in multiple fields
     @JsonIgnore
     private String searchableText;

diff --git a/.../java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java b/.../java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java
@@ -6,14 +6,20 @@
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetModel;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
 import it.gov.innovazione.ndc.harvester.model.extractors.RightsHolderExtractor;
+import it.gov.innovazione.ndc.harvester.model.index.NodeSummary;
 import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata;
 import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository;
 import it.gov.innovazione.ndc.repository.TripleStoreRepository;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.jena.rdf.model.Resource;
 
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+
 import static it.gov.innovazione.ndc.harvester.model.SemanticAssetModelValidationContext.NO_VALIDATION;
+import static java.util.stream.Collectors.toList;
 
 @RequiredArgsConstructor
 @Slf4j
@@ -67,9 +73,21 @@ protected void enrichModelBeforePersisting(M model, P path) {
     private void indexMetadataForSearch(M model) {
         log.debug("Indexing {} for search", model.getMainResource());
         SemanticAssetMetadata metadata = model.extractMetadata();
+        postProcessMetadata(metadata);
         metadataRepository.save(metadata);
     }
 
+    protected void postProcessMetadata(SemanticAssetMetadata metadata) {
+        if (Objects.nonNull(metadata)) {
+            metadata.setKeyClassesLabels(
+                    Optional.ofNullable(metadata)
+                            .map(SemanticAssetMetadata::getKeyClasses)
+                            .orElse(Collections.emptyList()).stream()
+                            .map(NodeSummary::getSummary)
+                            .collect(toList()));
+        }
+    }
+
     private void persistModelToTripleStore(String repoUrl, P path, M model) {
         log.debug("Storing RDF content for {} in Virtuoso", model.getMainResource());
         tripleStoreRepository.save(repoUrl, model.getRdfModel());

diff --git a/src/main/java/it/gov/innovazione/ndc/model/Builders.java b/src/main/java/it/gov/innovazione/ndc/model/Builders.java
@@ -1,33 +1,62 @@
 package it.gov.innovazione.ndc.model;
 
+import it.gov.innovazione.ndc.gen.dto.Direction;
 import it.gov.innovazione.ndc.gen.dto.Problem;
 import it.gov.innovazione.ndc.gen.dto.SearchResult;
 import it.gov.innovazione.ndc.gen.dto.SearchResultItem;
+import it.gov.innovazione.ndc.gen.dto.SortBy;
 import it.gov.innovazione.ndc.gen.dto.VocabulariesResult;
 import it.gov.innovazione.ndc.gen.dto.VocabularyData;
 import it.gov.innovazione.ndc.gen.dto.VocabularySummary;
 import it.gov.innovazione.ndc.gen.dto.VocabularySummaryLinks;
 import lombok.Builder;
 import lombok.SneakyThrows;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Sort;
 import org.springframework.http.HttpStatus;
 
 import java.net.URI;
 import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+
+import static com.github.jsonldjava.shaded.com.google.common.base.CaseFormat.LOWER_CAMEL;
+import static com.github.jsonldjava.shaded.com.google.common.base.CaseFormat.UPPER_UNDERSCORE;
 
 public class Builders {
     private Builders() {
     }
 
     @Builder(builderMethodName = "searchResult")
-    public static SearchResult newSearchResult(Integer totalCount, Integer limit, Integer offset, List<SearchResultItem> data) {
+    public static SearchResult newSearchResult(Integer totalCount, Integer limit, Integer offset, Pageable pageable, List<SearchResultItem> data) {
         SearchResult result = new SearchResult();
 
+        SortBy sortBy =
+                Optional.ofNullable(pageable)
+                        .map(Pageable::getSort)
+                        .flatMap(orders -> orders.stream()
+                                .findFirst()
+                                .map(Sort.Order::getProperty)
+                                .map(property -> LOWER_CAMEL.to(UPPER_UNDERSCORE, property))
+                                .map(SortBy::valueOf))
+                        .orElse(null);
+
+        Direction direction =
+                Optional.ofNullable(pageable)
+                        .map(Pageable::getSort)
+                        .flatMap(orders -> orders.stream()
+                                .findFirst()
+                                .map(Sort.Order::getDirection)
+                                .map(dir -> dir == Sort.Direction.ASC ? Direction.ASC : Direction.DESC))
+                        .orElse(null);
+
         result.setTotalCount(totalCount);
         result.setLimit(limit);
         result.setOffset(offset);
         result.setData(data);
+        result.setSortBy(sortBy);
+        result.setDirection(direction);
 
         return result;
     }

diff --git a/src/main/java/it/gov/innovazione/ndc/model/SemanticAssetsMetadataMapper.java b/src/main/java/it/gov/innovazione/ndc/model/SemanticAssetsMetadataMapper.java
@@ -54,6 +54,7 @@ default SearchResult searchResultToDto(SearchPage<SemanticAssetMetadata> source)
                 .totalCount((int) source.getTotalElements())
                 .limit(resultPage.getPageSize())
                 .offset((int) resultPage.getOffset())
+                .pageable(resultPage)
                 .data(source.getContent()
                         .stream()
                         .map(s -> resultItemToDto(s.getContent()))

diff --git a/src/main/resources/elasticsearch-settings.json b/src/main/resources/elasticsearch-settings.json
@@ -54,6 +54,14 @@
           "italian_stemmer"
         ]
       }
+    },
+    "normalizer": {
+      "lowercase_normalizer": {
+        "type": "custom",
+        "filter": [
+          "lowercase"
+        ]
+      }
     }
   }
-}
+}
diff --git a/src/main/resources/public/openapi.yaml b/src/main/resources/public/openapi.yaml
@@ -286,7 +286,7 @@ components:
       type: string
       enum:
         - TITLE
-        - LAST_MODIFIED
+        - MODIFIED_ON
       default: "TITLE"
     Direction:
       type: string

diff --git a/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java b/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java
@@ -1,7 +1,9 @@
 package it.gov.innovazione.ndc.controller;
 
+import it.gov.innovazione.ndc.gen.dto.Direction;
 import it.gov.innovazione.ndc.gen.dto.SearchResult;
 import it.gov.innovazione.ndc.gen.dto.SemanticAssetDetails;
+import it.gov.innovazione.ndc.gen.dto.SortBy;
 import it.gov.innovazione.ndc.gen.dto.Theme;
 import it.gov.innovazione.ndc.harvester.service.RepositoryService;
 import it.gov.innovazione.ndc.model.Builders;
@@ -10,6 +12,7 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.data.domain.Sort;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -37,7 +40,7 @@ void shouldFetchResultsFromRepositoryForGivenKeyword() {
 
         SearchResult actualResult =
             controller.search("searchTerm", 0, 10,
-                    null, null,
+                    SortBy.TITLE, Direction.ASC,
                 Set.of(CONTROLLED_VOCABULARY),
                     Set.of(Theme.EDUC),
                     Set.of()).getBody();
@@ -46,7 +49,7 @@ void shouldFetchResultsFromRepositoryForGivenKeyword() {
             Set.of("CONTROLLED_VOCABULARY"),
             Set.of("http://publications.europa.eu/resource/authority/data-theme/EDUC"),
                 Set.of(),
-            OffsetBasedPageRequest.of(0, 10));
+                OffsetBasedPageRequest.of(0, 10, Sort.by("title").ascending()));
         assertThat(actualResult).isEqualTo(expectedResult);
     }