From 25001b5466573d7adcdfe212d37fd5893fd3e23f Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Fri, 15 May 2020 22:40:43 +0200 Subject: [PATCH 1/6] specify dependencies instead of using starter --- build.gradle | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a2e3ffc7..9380526c 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'maven-publish' id 'co.uzzu.dotenv.gradle' version '1.1.0' id 'de.undercouch.download' version '4.0.4' - id 'org.springframework.boot' version '2.2.6.RELEASE' + id 'org.springframework.boot' version '2.3.0.RELEASE' id 'com.github.ben-manes.versions' version '0.28.0' id 'com.github.dawnwords.jacoco.badge' version '0.2.0' } @@ -56,9 +56,14 @@ dependencies { implementation 'com.github.jkatzwinkel:tla-common:master-SNAPSHOT' implementation 'org.modelmapper:modelmapper:2.3.7' implementation 'org.apache.commons:commons-compress:1.20' + implementation 'org.yaml:snakeyaml:1.25' implementation 'org.springframework.data:spring-data-elasticsearch:3.2.4.RELEASE' - implementation 'org.springframework.boot:spring-boot-starter-web:2.3.0.M1' + implementation 'org.springframework.boot:spring-boot:2.3.0.M1' + implementation 'org.springframework.boot:spring-boot-autoconfigure:2.3.0.M1' + implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.30' + implementation 'org.springframework:spring-web:5.2.6.RELEASE' + implementation 'org.springframework:spring-webmvc:5.2.6.RELEASE' implementation 'org.slf4j:slf4j-simple:2.0.0-alpha1' testImplementation 'org.springframework.boot:spring-boot-starter-test:2.2.4.RELEASE' From 0d6a47fddcd0cba98446ace617eaaa5688959c2b Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Sat, 16 May 2020 01:58:23 +0200 Subject: [PATCH 2/6] upgrade to elasticsearch 7.6.2 --- README.md | 6 +- build.gradle | 2 +- docker-compose.yml | 2 +- .../tla/backend/api/OccurrenceController.java | 10 ++- .../backend/es/model/OccurrenceEntity.java | 2 + .../backend/es/model/meta/ModelConfig.java | 5 ++ .../tla/backend/es/model/parts/EditDate.java | 86 +++++++++++++++++++ .../backend/es/model/parts/EditorInfo.java | 5 +- .../backend/es/model/parts/Transcription.java | 3 + .../java/tla/backend/es/repo/RepoConfig.java | 1 + .../tla/backend/es/repo/RepoPopulator.java | 14 +-- .../tla/backend/service/QueryService.java | 6 +- .../tla/backend/api/LemmaControllerTest.java | 14 ++- .../java/tla/backend/es/model/ModelTest.java | 49 +++++++---- .../java/tla/backend/es/repo/RepoTest.java | 20 ++++- .../object/2ABHCXHF5BCG3PDW7VSPA77K7U.json | 2 +- 16 files changed, 181 insertions(+), 46 deletions(-) create mode 100644 src/main/java/tla/backend/es/model/parts/EditDate.java diff --git a/README.md b/README.md index 8f009ae7..d48800fa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![Java CI](https://github.com/JKatzwinkel/tla-es/workflows/Java%20CI/badge.svg) -![LINE](https://img.shields.io/badge/line--coverage-56%25-orange.svg) -![METHOD](https://img.shields.io/badge/method--coverage-54%25-orange.svg) +![LINE](https://img.shields.io/badge/line--coverage-57%25-orange.svg) +![METHOD](https://img.shields.io/badge/method--coverage-55%25-orange.svg) Thesaurus Linguae Aegyptiae (TLA) backend. @@ -14,7 +14,7 @@ Requirements: ### 1. Install and populate search engine -The TLA backend app requires Elasticsearch 6.8.4. It can be installed via Docker Compose using the +The TLA backend app requires Elasticsearch 7.6.2. It can be installed via Docker Compose using the configuration shipped with this repository: docker-compose up -d elasticsearch diff --git a/build.gradle b/build.gradle index 9380526c..e9d47920 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,7 @@ dependencies { implementation 'org.apache.commons:commons-compress:1.20' implementation 'org.yaml:snakeyaml:1.25' - implementation 'org.springframework.data:spring-data-elasticsearch:3.2.4.RELEASE' + implementation 'org.springframework.data:spring-data-elasticsearch:4.0.0.RELEASE' implementation 'org.springframework.boot:spring-boot:2.3.0.M1' implementation 'org.springframework.boot:spring-boot-autoconfigure:2.3.0.M1' implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.30' diff --git a/docker-compose.yml b/docker-compose.yml index 5008fbd2..d78cf9ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ version: '3.7' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.8.4 + image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 environment: ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node diff --git a/src/main/java/tla/backend/api/OccurrenceController.java b/src/main/java/tla/backend/api/OccurrenceController.java index 913d58c6..03b03855 100644 --- a/src/main/java/tla/backend/api/OccurrenceController.java +++ b/src/main/java/tla/backend/api/OccurrenceController.java @@ -8,7 +8,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; -import org.springframework.data.elasticsearch.core.query.SearchQuery; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.BucketOrder; @@ -47,13 +48,16 @@ public class OccurrenceController { @Autowired private ElasticsearchRestTemplate restTemplate; + @Autowired + private RestHighLevelClient restClient; + @Autowired private LemmaService lemmaService; @RequestMapping(method = RequestMethod.GET, value = "/lemma/{id}") public ResponseEntity> lemmaOccurrences(@PathVariable String id) { - SearchQuery query = new NativeSearchQueryBuilder().withQuery(termQuery("lemma.id", id)).build(); + NativeSearchQuery query = new NativeSearchQueryBuilder().withQuery(termQuery("lemma.id", id)).build(); long occurrenceCount = restTemplate.count(query, OccurrenceEntity.class); return new ResponseEntity>(Map.of(id, occurrenceCount), HttpStatus.OK); } @@ -83,7 +87,7 @@ public ResponseEntity> frequencies() throws IOException { ) .source(searchSourceBuilder) .requestCache(true); - SearchResponse response = restTemplate.getClient().search( + SearchResponse response = restClient.search( request, RequestOptions.DEFAULT ); diff --git a/src/main/java/tla/backend/es/model/OccurrenceEntity.java b/src/main/java/tla/backend/es/model/OccurrenceEntity.java index 720684bc..7b193b85 100644 --- a/src/main/java/tla/backend/es/model/OccurrenceEntity.java +++ b/src/main/java/tla/backend/es/model/OccurrenceEntity.java @@ -1,6 +1,7 @@ package tla.backend.es.model; import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.springframework.data.annotation.Id; import org.springframework.data.elasticsearch.annotations.Document; @@ -24,6 +25,7 @@ @Builder @NoArgsConstructor @AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) @Document(indexName = "occurrence", type = "occurrence") public class OccurrenceEntity implements Indexable { diff --git a/src/main/java/tla/backend/es/model/meta/ModelConfig.java b/src/main/java/tla/backend/es/model/meta/ModelConfig.java index cd31b3f7..00641451 100644 --- a/src/main/java/tla/backend/es/model/meta/ModelConfig.java +++ b/src/main/java/tla/backend/es/model/meta/ModelConfig.java @@ -21,6 +21,7 @@ import tla.backend.es.model.LemmaEntity; import tla.backend.es.model.TextEntity; import tla.backend.es.model.ThsEntryEntity; +import tla.backend.es.model.parts.EditorInfo; import tla.backend.es.model.parts.Translations; import tla.domain.dto.AnnotationDto; import tla.domain.dto.CorpusObjectDto; @@ -245,6 +246,10 @@ private static ModelMapper initModelMapper() { log.debug("initializing model mapper"); modelMapper = new ModelMapper(); Translations.ToMapConverter translationsToMapConverter = new Translations.ToMapConverter(); + modelMapper.createTypeMap(EditorInfo.class, tla.domain.model.EditorInfo.class) + .addMapping( + EditorInfo::getUpdated, tla.domain.model.EditorInfo::setDateOfLatestUpdate + ); modelMapper.createTypeMap(LemmaEntity.class, LemmaDto.class) .addMappings( m -> m.using(translationsToMapConverter).map( diff --git a/src/main/java/tla/backend/es/model/parts/EditDate.java b/src/main/java/tla/backend/es/model/parts/EditDate.java new file mode 100644 index 00000000..51b5e062 --- /dev/null +++ b/src/main/java/tla/backend/es/model/parts/EditDate.java @@ -0,0 +1,86 @@ +package tla.backend.es.model.parts; + +import java.io.Serializable; +import java.time.LocalDate; +import java.time.chrono.ChronoLocalDate; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalField; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import org.springframework.data.elasticsearch.core.convert.ElasticsearchDateConverter; + +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode +public class EditDate implements TemporalAccessor, Serializable, Comparable { + + public static ElasticsearchDateConverter DATE_CONVERTER = ElasticsearchDateConverter.of( + "yyyy-MM-dd" + ); + + private static final long serialVersionUID = -2478025723214725811L; + + private LocalDate date; + + public EditDate(LocalDate date) { + this.date = date; + } + + @JsonCreator + public static EditDate fromString(String source) throws Exception { + String[] coords = source.split("-"); + if (coords.length == 3) { + int year = Integer.valueOf(coords[0]); + int mon = Integer.valueOf(coords[1]); + int day = coords[2].length() < 3 ? Integer.valueOf(coords[2]) : Integer.valueOf(coords[2].split(" ")[0]); + return new EditDate(LocalDate.of(year, mon, day)); + } else { + throw new Exception(String.format("could not create date instance from input '%s'", source)); + } + } + + @JsonValue + @EqualsAndHashCode.Include + public String toString() { + return DATE_CONVERTER.format(date); + } + + public static EditDate of(int year, int month, int day) { + return new EditDate( + LocalDate.of(year, month, day) + ); + } + + public static EditDate from(TemporalAccessor temporal) { + return new EditDate( + LocalDate.of( + temporal.get(ChronoField.YEAR_OF_ERA), + temporal.get(ChronoField.MONTH_OF_YEAR), + temporal.get(ChronoField.DAY_OF_MONTH) + ) + ); + } + + @Override + public long getLong(TemporalField arg0) { + if (this.date.isSupported(arg0)) { + return this.date.getLong(arg0); + } else { + return 0; + } + } + + @Override + public boolean isSupported(TemporalField arg0) { + return date.isSupported(arg0); + } + + @Override + public int compareTo(ChronoLocalDate arg0) { + return date.compareTo(arg0); + } + +} \ No newline at end of file diff --git a/src/main/java/tla/backend/es/model/parts/EditorInfo.java b/src/main/java/tla/backend/es/model/parts/EditorInfo.java index 7ee3f46e..b56603cd 100644 --- a/src/main/java/tla/backend/es/model/parts/EditorInfo.java +++ b/src/main/java/tla/backend/es/model/parts/EditorInfo.java @@ -1,6 +1,5 @@ package tla.backend.es.model.parts; -import java.util.Date; import java.util.List; import com.fasterxml.jackson.annotation.JsonFormat; @@ -27,9 +26,9 @@ public class EditorInfo { @Field(type = FieldType.Text) private String author; - @Field(type = FieldType.Date, format = DateFormat.custom, pattern = "yyyy-MM-dd||epoch_millis") + @Field(type = FieldType.Date, format = DateFormat.custom, pattern = "yyyy-MM-dd") @JsonFormat(pattern = "yyyy-MM-dd", timezone = "UTC") - private Date updated; + private EditDate updated; @Field(type = FieldType.Keyword) private String type; diff --git a/src/main/java/tla/backend/es/model/parts/Transcription.java b/src/main/java/tla/backend/es/model/parts/Transcription.java index 0dfa36a2..8631240f 100644 --- a/src/main/java/tla/backend/es/model/parts/Transcription.java +++ b/src/main/java/tla/backend/es/model/parts/Transcription.java @@ -1,5 +1,7 @@ package tla.backend.es.model.parts; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; @@ -12,6 +14,7 @@ @Builder @NoArgsConstructor @AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) public class Transcription { @Field(type = FieldType.Text) diff --git a/src/main/java/tla/backend/es/repo/RepoConfig.java b/src/main/java/tla/backend/es/repo/RepoConfig.java index a1cbf235..5ef73b80 100644 --- a/src/main/java/tla/backend/es/repo/RepoConfig.java +++ b/src/main/java/tla/backend/es/repo/RepoConfig.java @@ -32,6 +32,7 @@ public ElasticsearchRestTemplate elasticsearchRestTemplate() { ); } + @Bean @Override public RestHighLevelClient elasticsearchClient() { log.info diff --git a/src/main/java/tla/backend/es/repo/RepoPopulator.java b/src/main/java/tla/backend/es/repo/RepoPopulator.java index 798d8600..43932e7d 100644 --- a/src/main/java/tla/backend/es/repo/RepoPopulator.java +++ b/src/main/java/tla/backend/es/repo/RepoPopulator.java @@ -7,11 +7,12 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.elasticsearch.core.EntityMapper; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import lombok.extern.slf4j.Slf4j; @@ -28,11 +29,15 @@ public class RepoPopulator { private class RepoBatchIngestor, S extends Indexable> { + final static int MAX_BATCH_SIZE = 750; + private List batch; private T repo; private Class modelClass; private int count; + private ObjectMapper jsonMapper = new ObjectMapper(); + public RepoBatchIngestor(T repo, Class modelClass) { this.repo = repo; log.info("modelclass: {}, repo: {}", modelClass.getName(), repo); @@ -43,14 +48,14 @@ public RepoBatchIngestor(T repo, Class modelClass) { public void add(S doc) { this.batch.add(doc); - if (this.batch.size() > 1000) { + if (this.batch.size() >= MAX_BATCH_SIZE) { this.ingest(); } } public void add(String json) { try { - S doc = mapper.mapToObject( + S doc = jsonMapper.readValue( json, this.modelClass ); @@ -86,9 +91,6 @@ public void ingest() { } } - @Autowired - private EntityMapper mapper; - @Autowired private LemmaRepo lemmaRepo; diff --git a/src/main/java/tla/backend/service/QueryService.java b/src/main/java/tla/backend/service/QueryService.java index b90fd0f1..225e3905 100644 --- a/src/main/java/tla/backend/service/QueryService.java +++ b/src/main/java/tla/backend/service/QueryService.java @@ -16,8 +16,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Service; @@ -40,7 +40,7 @@ public abstract class QueryService { @Autowired - private ElasticsearchRestTemplate restTemplate; + private RestHighLevelClient restClient; protected static Map> eclassServices = new HashMap<>(); @@ -211,7 +211,7 @@ public SearchResponse query( ); SearchResponse response = null; try { - response = restTemplate.getClient() + response = restClient .search( request, RequestOptions.DEFAULT diff --git a/src/test/java/tla/backend/api/LemmaControllerTest.java b/src/test/java/tla/backend/api/LemmaControllerTest.java index f9556a5d..48f599b7 100644 --- a/src/test/java/tla/backend/api/LemmaControllerTest.java +++ b/src/test/java/tla/backend/api/LemmaControllerTest.java @@ -7,11 +7,10 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.data.elasticsearch.core.EntityMapper; import tla.backend.AbstractMockMvcTest; -import tla.backend.Util; import tla.backend.es.model.LemmaEntity; +import tla.backend.es.model.parts.EditDate; import tla.backend.es.model.parts.EditorInfo; import tla.backend.es.model.parts.LemmaWord; import tla.backend.es.model.parts.Transcription; @@ -31,8 +30,7 @@ public class LemmaControllerTest extends AbstractMockMvcTest { @MockBean private LemmaRepo repo; - @Autowired - private EntityMapper mapper; + private ObjectMapper mapper = new ObjectMapper(); @Autowired private LemmaService service; @@ -53,7 +51,7 @@ void nonNullFieldValidation() { @Test void mapLemma() throws Exception { assertNotNull(mapper, "elasticsearch jackson-based mapper should not be null"); - LemmaEntity l = mapper.mapToObject("{\"id\":\"ID\",\"sort_string\":\"1\"}", LemmaEntity.class); + LemmaEntity l = mapper.readValue("{\"id\":\"ID\",\"sort_string\":\"1\"}", LemmaEntity.class); assertEquals("1", l.getSortKey(), "sort_string should be deserialized correctly"); } @@ -62,15 +60,15 @@ void mapLemmaEquals() throws Exception { LemmaEntity l1 = LemmaEntity.builder() .id("1") .eclass("BTSLemmaEntry") - .editors(EditorInfo.builder().author("author").updated(Util.date("1854-10-31")).build()) + .editors(EditorInfo.builder().author("author").updated(EditDate.of(1854,10,31)).build()) .translations(Translations.builder().de("übersetzung").build()) .word(new LemmaWord("N35:G47", new Transcription("nfr", "nfr"))) .build(); String ser = new ObjectMapper().writeValueAsString(l1); - LemmaEntity l2 = mapper.mapToObject(ser, LemmaEntity.class); + LemmaEntity l2 = mapper.readValue(ser, LemmaEntity.class); assertAll("lemma instance created via lombok builder should be the same after being serialized by object mapper and deserialized by ES entity mapper", () -> assertEquals(l1, l2, "equals should return true"), - () -> assertEquals(mapper.mapToString(l1), mapper.mapToString(l2), "ES entity mapper serializations should be equal"), + () -> assertEquals(mapper.writeValueAsString(l1), mapper.writeValueAsString(l2), "ES entity mapper serializations should be equal"), () -> assertEquals(l1.hashCode(), l2.hashCode(), "hashCode should return equals"), () -> assertNotNull(l2.getWords().get(0).getTranscription(), "expect word transcription") ); diff --git a/src/test/java/tla/backend/es/model/ModelTest.java b/src/test/java/tla/backend/es/model/ModelTest.java index 6fcd0e25..1c5e703f 100644 --- a/src/test/java/tla/backend/es/model/ModelTest.java +++ b/src/test/java/tla/backend/es/model/ModelTest.java @@ -4,18 +4,18 @@ import java.util.Map; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.data.elasticsearch.annotations.Document; -import org.springframework.data.elasticsearch.core.EntityMapper; import tla.backend.App; -import tla.backend.Util; import tla.backend.es.model.meta.BaseEntity; import tla.backend.es.model.meta.Indexable; import tla.backend.es.model.meta.ModelConfig; import tla.backend.es.model.meta.TLAEntity; +import tla.backend.es.model.parts.EditDate; import tla.backend.es.model.parts.EditorInfo; import tla.backend.es.model.parts.LemmaWord; import tla.backend.es.model.parts.Transcription; @@ -37,8 +37,7 @@ @SpringBootTest(classes = {App.class}) public class ModelTest { - @Autowired - private EntityMapper mapper; + private ObjectMapper mapper = new ObjectMapper(); @Test void modelConfigInitialized() { @@ -124,7 +123,7 @@ void translationsEqual() throws Exception { assertTrue(mapper != null, "entitymapper should not be null"); Translations t1 = Translations.builder().de("übersetzung").en("translation").en("meaning").build(); Translations t2 = Translations.builder().de(Arrays.asList("übersetzung")).en(Arrays.asList("translation", "meaning")).build(); - Translations t3 = mapper.mapToObject("{\"de\": [\"übersetzung\"], \"en\": [\"translation\", \"meaning\"]}", Translations.class); + Translations t3 = mapper.readValue("{\"de\": [\"übersetzung\"], \"en\": [\"translation\", \"meaning\"]}", Translations.class); assertAll("translations objects should be equal", () -> assertEquals(t2, t1, "translation instances should be equal regardless of build method parameter type"), () -> assertEquals(t3, t1, "deserialized instance should be equal to builder-instantiated"), @@ -133,19 +132,35 @@ void translationsEqual() throws Exception { ); } + @Test + void editInfoEqual() throws Exception { + EditorInfo e1 = mapper.readValue( + "{\"author\":\"author\", \"updated\":\"2019-12-18\"}", + EditorInfo.class + ); + EditorInfo e2 = mapper.readValue( + mapper.writeValueAsString(e1), + EditorInfo.class + ); + assertAll("test EditorInfo equality", + () -> assertEquals(e1, e2, "instances as a whole should be equal"), + () -> assertEquals(e1.getUpdated(), e2.getUpdated(), "edit dates should be equals") + ); + } + @Test void thesaurusEntriesEqual() throws Exception { ThsEntryEntity t_built = ThsEntryEntity.builder() .id("1") .eclass("BTSThsEntry") .sortKey("1") - .editors(EditorInfo.builder().author("author").updated(Util.date("2015-12-31")).build()) + .editors(EditorInfo.builder().author("author").updated(EditDate.of(2015, 12, 31)).build()) .build(); - ThsEntryEntity t_read = mapper.mapToObject( + ThsEntryEntity t_read = mapper.readValue( "{\"id\":\"ID\",\"eclass\":\"BTSThsEntry\",\"sort_string\":\"1\",\"editors\":{\"author\":\"author\",\"updated\":\"2015-12-31\"}}", ThsEntryEntity.class ); - ThsEntryEntity t_round = mapper.mapToObject(mapper.mapToString(t_built), ThsEntryEntity.class); + ThsEntryEntity t_round = mapper.readValue(mapper.writeValueAsString(t_built), ThsEntryEntity.class); assertAll("thesaurus entry instances should be equal regardless of creation method", () -> assertNotEquals(t_built, t_read, "deserialized instance should not be the same as built instance"), () -> assertEquals(t_built, t_round, "built instance should remain the same after serialization and deserialization via ES entity mapper"), @@ -188,12 +203,12 @@ void lemmaEntriesEqual() throws Exception { .id("1") .passport(new Passport()) .build(); - LemmaEntity l_read = mapper.mapToObject( + LemmaEntity l_read = mapper.readValue( "{\"id\":\"1\",\"passport\":{}}", LemmaEntity.class ); - LemmaEntity l_round = mapper.mapToObject( - mapper.mapToString(l_built), + LemmaEntity l_round = mapper.readValue( + mapper.writeValueAsString(l_built), LemmaEntity.class ); assertAll("lemma entry instances should be equal regardless of creation method", @@ -284,8 +299,8 @@ void annotationDeserializeFromFile() throws Exception { () -> assertEquals(1, a.getPassport().extractProperty("annotation.lemma").size()), () -> assertTrue(a.getPassport().extractProperty("annotation.lemma").get(0).getLeafNodeValue().length() > 100) ); - AnnotationEntity b = mapper.mapToObject( - mapper.mapToString(a), + AnnotationEntity b = mapper.readValue( + mapper.writeValueAsString(a), AnnotationEntity.class ); assertAll("after serialization and deserialization, annotation object should be still the same", @@ -334,6 +349,8 @@ void corpusobjectFromFileMapping() throws Exception { ); assertAll("test corpus object deserialization from JSON file", () -> assertNotNull(o), + () -> assertNotNull(o.getEditors(), "editor info not null"), + () -> assertNotNull(o.getEditors().getUpdated(), "edit date no null"), () -> assertNotNull(o.getPaths()), () -> assertEquals("bbawarchive", o.getCorpus()) ); @@ -342,7 +359,9 @@ void corpusobjectFromFileMapping() throws Exception { () -> assertNotNull(d), () -> assertTrue(d instanceof CorpusObjectDto), () -> assertNotNull(((CorpusObjectDto) d).getPaths()), - () -> assertEquals(o.getRevisionState(), d.getReviewState()) + () -> assertEquals(o.getRevisionState(), d.getReviewState()), + () -> assertNotNull(d.getEditors(), "DTO edit info not null"), + () -> assertNotNull(d.getEditors().getUpdated(), "DTO edit date not null") ); } diff --git a/src/test/java/tla/backend/es/repo/RepoTest.java b/src/test/java/tla/backend/es/repo/RepoTest.java index 809715fa..7d806584 100644 --- a/src/test/java/tla/backend/es/repo/RepoTest.java +++ b/src/test/java/tla/backend/es/repo/RepoTest.java @@ -8,8 +8,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchDateConverter; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import tla.backend.App; +import tla.backend.es.model.parts.EditDate; @SpringBootTest(classes = {App.class}) public class RepoTest { @@ -32,13 +36,25 @@ void esRestTemplateAvailable() { @Test void indexLemmaSetup() { - Map lemmaIndexSettings = elasticsearchRestTemplate.getSetting("lemma"); + IndexOperations index = elasticsearchRestTemplate.indexOps( + IndexCoordinates.of("lemma") + ); + Map lemmaIndexSettings = index.getSettings(); assertAll("index lemma should exist and be configured correctly", - () -> assertTrue(elasticsearchRestTemplate.indexExists("lemma"), "index lemma should exist"), + () -> assertTrue(index.exists(), "index lemma should exist"), () -> assertTrue(!lemmaIndexSettings.isEmpty(), "lemma index should contain settings"), () -> assertTrue(lemmaIndexSettings.containsKey("index.max_result_window"), "settings should contain key `max_result_window`"), () -> assertEquals("100000", lemmaIndexSettings.get("index.max_result_window")) ); } + @Test + void esDateConverterShouldParseLocalDate() { + ElasticsearchDateConverter converter = ElasticsearchDateConverter.of("yyyy-MM-dd"); + EditDate localDate = converter.parse("2019-12-18", EditDate.class); + assertAll("", + () -> assertNotNull(localDate), + () -> assertEquals("2019-12-18", localDate.toString()) + ); + } } \ No newline at end of file diff --git a/src/test/resources/sample/object/2ABHCXHF5BCG3PDW7VSPA77K7U.json b/src/test/resources/sample/object/2ABHCXHF5BCG3PDW7VSPA77K7U.json index bc3ca4e3..ee02a7ed 100644 --- a/src/test/resources/sample/object/2ABHCXHF5BCG3PDW7VSPA77K7U.json +++ b/src/test/resources/sample/object/2ABHCXHF5BCG3PDW7VSPA77K7U.json @@ -7,7 +7,7 @@ "Altägyptisches Wörterbuch" ], "type": "user", - "updated": "2019-11-14 22:10:09" + "updated": "2019-11-14" }, "externalReferences": null, "id": "2ABHCXHF5BCG3PDW7VSPA77K7U", From 091d81244a6723ead0f99f0a1241db00b868b735 Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Sat, 16 May 2020 02:10:57 +0200 Subject: [PATCH 3/6] update dependencies --- build.gradle | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index e9d47920..d64f5838 100644 --- a/build.gradle +++ b/build.gradle @@ -56,17 +56,17 @@ dependencies { implementation 'com.github.jkatzwinkel:tla-common:master-SNAPSHOT' implementation 'org.modelmapper:modelmapper:2.3.7' implementation 'org.apache.commons:commons-compress:1.20' - implementation 'org.yaml:snakeyaml:1.25' + implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.30' + implementation 'org.yaml:snakeyaml:1.26' implementation 'org.springframework.data:spring-data-elasticsearch:4.0.0.RELEASE' - implementation 'org.springframework.boot:spring-boot:2.3.0.M1' - implementation 'org.springframework.boot:spring-boot-autoconfigure:2.3.0.M1' - implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.30' + implementation 'org.springframework.boot:spring-boot:2.3.0.RELEASE' + implementation 'org.springframework.boot:spring-boot-autoconfigure:2.3.0.RELEASE' implementation 'org.springframework:spring-web:5.2.6.RELEASE' implementation 'org.springframework:spring-webmvc:5.2.6.RELEASE' implementation 'org.slf4j:slf4j-simple:2.0.0-alpha1' - testImplementation 'org.springframework.boot:spring-boot-starter-test:2.2.4.RELEASE' + testImplementation 'org.springframework.boot:spring-boot-starter-test:2.3.0.RELEASE' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0-M1' } From fad96b7374fbf15a9f6cb6f5b6bb4b5dd82514a8 Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Sat, 16 May 2020 03:07:40 +0200 Subject: [PATCH 4/6] update embedded tomcat --- .gitignore | 5 +++++ build.gradle | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 789f6ffa..57241aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,8 @@ bin .env settings.gradle .settings +.gradletasknamecache +.project +sample.tar.gz +app.log +app.log.* diff --git a/build.gradle b/build.gradle index d64f5838..a5535fb3 100644 --- a/build.gradle +++ b/build.gradle @@ -56,7 +56,7 @@ dependencies { implementation 'com.github.jkatzwinkel:tla-common:master-SNAPSHOT' implementation 'org.modelmapper:modelmapper:2.3.7' implementation 'org.apache.commons:commons-compress:1.20' - implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.30' + implementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.35' implementation 'org.yaml:snakeyaml:1.26' implementation 'org.springframework.data:spring-data-elasticsearch:4.0.0.RELEASE' From f2601fb2c6e607e3e12b83c807d08e61156c4854 Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Sat, 16 May 2020 03:53:04 +0200 Subject: [PATCH 5/6] sneaky test to get notified of future fixes of date/localdate conversion issues in data es --- src/test/java/tla/backend/es/repo/RepoTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/java/tla/backend/es/repo/RepoTest.java b/src/test/java/tla/backend/es/repo/RepoTest.java index 7d806584..cdeabf21 100644 --- a/src/test/java/tla/backend/es/repo/RepoTest.java +++ b/src/test/java/tla/backend/es/repo/RepoTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.*; +import java.time.DateTimeException; +import java.time.LocalDate; import java.util.Map; import org.junit.jupiter.api.Test; @@ -9,6 +11,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.convert.ConversionException; import org.springframework.data.elasticsearch.core.convert.ElasticsearchDateConverter; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -56,5 +59,11 @@ void esDateConverterShouldParseLocalDate() { () -> assertNotNull(localDate), () -> assertEquals("2019-12-18", localDate.toString()) ); + assertThrows(DateTimeException.class, + () -> converter.parse("2019-12-18") + ); + assertThrows(ConversionException.class, + () -> converter.parse("2019-12-18", LocalDate.class) + ); } } \ No newline at end of file From 55ce8d5448b2dc92d053c5815d4b64ed5cdd1f35 Mon Sep 17 00:00:00 2001 From: JKatzwinkel Date: Sat, 16 May 2020 15:44:11 +0200 Subject: [PATCH 6/6] update version number to reflect ES upgrade --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a5535fb3..70f0a332 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ plugins { } group = 'org.bbaw.aaew.tla' -version = '0.0.1-SNAPSHOT' +version = '0.0.7-SNAPSHOT' sourceCompatibility = '11' publishing {