From 6be303e4105c5eb9b5c5f54520759d00c553a301 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Wed, 30 Oct 2019 17:57:52 +0100 Subject: [PATCH 01/10] draft --- .../ingest/common/DateFormat.java | 8 +++---- qa/mixed-cluster/build.gradle | 4 ++++ .../search/180_locale_dependent_mapping.yml | 3 +-- server/build.gradle | 3 ++- .../index/NodeMappingRefreshAction.java | 1 + .../PublicationTransportHandler.java | 4 ++-- .../metadata/MetaDataMappingService.java | 6 ++++++ .../common/time/DateFormatter.java | 21 +++++++++++-------- .../zen/PublishClusterStateAction.java | 6 +++--- .../index/mapper/DocumentMapper.java | 17 +++++++++++++++ .../index/mapper/MapperService.java | 6 ++++++ 11 files changed, 58 insertions(+), 21 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index c4b96bb06b0b2..8ff1b4664fc67 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -82,10 +82,10 @@ private long parseMillis(String date) { @Override Function getFunction(String format, ZoneId zoneId, Locale locale) { - // support the 6.x BWC compatible way of parsing java 8 dates - if (format.startsWith("8")) { - format = format.substring(1); - } + // support the 6.x BWC compatible way of parsing java 8 dates //TODO this is not needed?? +// if (format.startsWith("8")) { +// format = format.substring(1); +// } boolean isUtc = ZoneOffset.UTC.equals(zoneId); diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 5f3690c293d85..7f13b10efc06b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -75,8 +75,12 @@ for (Version bwcVersion : bwcVersions.wireCompatible) { println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",") }" println "Upgrading another node to create a mixed cluster" testClusters."${baseName}".nextNodeToNextVersion() +// testClusters."${baseName}".nextNodeToNextVersion() +// testClusters."${baseName}".nextNodeToNextVersion() + println "${-> testClusters."${baseName}".allHttpSocketURI[6]}" nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",") }") +// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI[4]}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName() }") } systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 0bd139a070bc2..d841229f306c4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -2,7 +2,6 @@ "Test Index and Search locale dependent mappings / dates": - skip: version: "all" - reason: "Awaits fix: https://github.com/elastic/elasticsearch/issues/39981(Previously: JDK9 only supports this with a special sysproperty added in 6.2.0.)" - do: indices.create: index: test_index @@ -13,7 +12,7 @@ properties: date_field: type: date - format: "E, d MMM yyyy HH:mm:ss Z" + format: "8E, d MMM yyyy HH:mm:ss Z" locale: "de" - do: bulk: diff --git a/server/build.gradle b/server/build.gradle index 551b148456765..1eeaa89df8521 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -98,7 +98,8 @@ dependencies { compile "org.apache.lucene:lucene-spatial-extras:${versions.lucene}" compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}" compile "org.apache.lucene:lucene-suggest:${versions.lucene}" - + compile "commons-lang:commons-lang:2.6" + // utilities compile project(":libs:elasticsearch-cli") compile 'com.carrotsearch:hppc:0.8.1' diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java index 5b377989c6530..ffccb0bbe96c6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java @@ -62,6 +62,7 @@ public void nodeMappingRefresh(final DiscoveryNode masterNode, final NodeMapping logger.warn("can't send mapping refresh for [{}], no master known.", request.index()); return; } +// logger.info("refresh "+request.index +" "+request.indexUUID+" "+request.nodeId); transportService.sendRequest(masterNode, ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 504d6be126d2a..062b4c6032358 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -425,8 +425,8 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque throw e; } compatibleClusterStateDiffReceivedCount.incrementAndGet(); - logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", - incomingState.version(), incomingState.stateUUID(), request.bytes().length()); +// logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}] {}", +// incomingState.version(), incomingState.stateUUID(), request.bytes().length(), incomingState); final PublishWithJoinResponse response = acceptState(incomingState); lastSeenClusterState.compareAndSet(lastSeen, incomingState); return response; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 4dc38c97a8e38..e111bc3f73016 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -105,6 +105,8 @@ public ClusterTasksResult execute(ClusterState currentState, List allTasks) throws Exception { +// logger.info("refresh "+currentState +" "+allTasks); + // break down to tasks per index, so we can optimize the on demand index service creation // to only happen for the duration of a single index processing of its respective events Map> tasksPerIndex = new HashMap<>(); @@ -154,6 +156,8 @@ ClusterState executeRefresh(final ClusterState currentState, final List patterns = splitCombinedPatterns(input); + String format = strip8Prefix(input); + List patterns = splitCombinedPatterns(format); List formatters = patterns.stream() .map(DateFormatters::forPattern) .collect(Collectors.toList()); - if (formatters.size() == 1) { - return formatters.get(0); - } + return JavaDateFormatter.combined(format, formatters); + } - return JavaDateFormatter.combined(input, formatters); + static String strip8Prefix(String input) { + if (input.startsWith("8")) { + return input.substring(1); + } + return input; } static List splitCombinedPatterns(String input) { diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java index af3174bd99c86..1327c78f9e77b 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; +import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.bytes.BytesReference; @@ -45,7 +46,6 @@ import org.elasticsearch.discovery.BlockingClusterStatePublishResponseHandler; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.BytesTransportRequest; @@ -396,8 +396,8 @@ protected void handleIncomingClusterStateRequest(BytesTransportRequest request, Diff diff = ClusterState.readDiffFrom(in, lastSeenClusterState.nodes().getLocalNode()); incomingState = diff.apply(lastSeenClusterState); compatibleClusterStateDiffReceivedCount.incrementAndGet(); - logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", - incomingState.version(), incomingState.stateUUID(), request.bytes().length()); +// logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}] {}", +// incomingState.version(), incomingState.stateUUID(), request.bytes().length(), incomingState); } else { logger.debug("received diff for but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 8de282787cc8a..467e31638bc2e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -339,4 +339,21 @@ public DocumentMapper updateFieldType(Map fullNameToFie public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return mapping.toXContent(builder, params); } + + @Override + public String toString() { + return "DocumentMapper{" + + "mapperService=" + mapperService + + ", type='" + type + '\'' + + ", typeText=" + typeText + + ", mappingSource=" + mappingSource + + ", mapping=" + mapping + + ", documentParser=" + documentParser + + ", fieldMappers=" + fieldMappers + + ", objectMappers=" + objectMappers + + ", hasNestedObjects=" + hasNestedObjects + + ", deleteTombstoneMetadataFieldMappers=" + Arrays.toString(deleteTombstoneMetadataFieldMappers) + + ", noopTombstoneMetadataFieldMappers=" + Arrays.toString(noopTombstoneMetadataFieldMappers) + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 57d8f69709001..fe70efb9ad1d2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -270,6 +270,10 @@ private void assertMappingVersion( if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) { // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same assert updatedEntries.isEmpty() : updatedEntries; + /* + if(!updatedEntries.isEmpty()) + logger.info("updated entries not empty" + updatedEntries); + */ MappingMetaData defaultMapping = newIndexMetaData.defaultMapping(); if (defaultMapping != null) { @@ -347,6 +351,8 @@ private synchronized Map internalMerge(IndexMetaData ind if (onlyUpdateIfNeeded) { DocumentMapper existingMapper = documentMapper(mappingMetaData.type()); if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) { + logger.info("he "+mappingMetaData.source()); + logger.info("he2 "+(existingMapper== null || existingMapper.mappingSource() == null ? "null" : existingMapper.mappingSource())); map.put(mappingMetaData.type(), mappingMetaData.source()); } } else { From 54c265e9186ef04f22870b53f7a51e9687fc5bdf Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 31 Oct 2019 09:46:20 +0100 Subject: [PATCH 02/10] remove unused logging --- qa/mixed-cluster/build.gradle | 3 --- .../cluster/action/index/NodeMappingRefreshAction.java | 1 - .../cluster/coordination/PublicationTransportHandler.java | 4 ++-- .../cluster/metadata/MetaDataMappingService.java | 5 ----- .../java/org/elasticsearch/common/time/DateFormatter.java | 2 -- .../discovery/zen/PublishClusterStateAction.java | 4 ++-- .../java/org/elasticsearch/index/mapper/MapperService.java | 4 ---- 7 files changed, 4 insertions(+), 19 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 7f13b10efc06b..9e3227de57dcf 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -75,12 +75,9 @@ for (Version bwcVersion : bwcVersions.wireCompatible) { println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",") }" println "Upgrading another node to create a mixed cluster" testClusters."${baseName}".nextNodeToNextVersion() -// testClusters."${baseName}".nextNodeToNextVersion() -// testClusters."${baseName}".nextNodeToNextVersion() println "${-> testClusters."${baseName}".allHttpSocketURI[6]}" nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",") }") -// nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI[4]}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName() }") } systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java index ffccb0bbe96c6..5b377989c6530 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java @@ -62,7 +62,6 @@ public void nodeMappingRefresh(final DiscoveryNode masterNode, final NodeMapping logger.warn("can't send mapping refresh for [{}], no master known.", request.index()); return; } -// logger.info("refresh "+request.index +" "+request.indexUUID+" "+request.nodeId); transportService.sendRequest(masterNode, ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 062b4c6032358..504d6be126d2a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -425,8 +425,8 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque throw e; } compatibleClusterStateDiffReceivedCount.incrementAndGet(); -// logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}] {}", -// incomingState.version(), incomingState.stateUUID(), request.bytes().length(), incomingState); + logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", + incomingState.version(), incomingState.stateUUID(), request.bytes().length()); final PublishWithJoinResponse response = acceptState(incomingState); lastSeenClusterState.compareAndSet(lastSeen, incomingState); return response; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index e111bc3f73016..2f337406f2679 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -105,8 +105,6 @@ public ClusterTasksResult execute(ClusterState currentState, List allTasks) throws Exception { -// logger.info("refresh "+currentState +" "+allTasks); - // break down to tasks per index, so we can optimize the on demand index service creation // to only happen for the duration of a single index processing of its respective events Map> tasksPerIndex = new HashMap<>(); @@ -156,7 +154,6 @@ ClusterState executeRefresh(final ClusterState currentState, final List diff = ClusterState.readDiffFrom(in, lastSeenClusterState.nodes().getLocalNode()); incomingState = diff.apply(lastSeenClusterState); compatibleClusterStateDiffReceivedCount.incrementAndGet(); -// logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}] {}", -// incomingState.version(), incomingState.stateUUID(), request.bytes().length(), incomingState); + logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", + incomingState.version(), incomingState.stateUUID(), request.bytes().length()); } else { logger.debug("received diff for but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index fe70efb9ad1d2..aea9840279de8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -270,10 +270,6 @@ private void assertMappingVersion( if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) { // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same assert updatedEntries.isEmpty() : updatedEntries; - /* - if(!updatedEntries.isEmpty()) - logger.info("updated entries not empty" + updatedEntries); - */ MappingMetaData defaultMapping = newIndexMetaData.defaultMapping(); if (defaultMapping != null) { From ca3f2afec7da0e9922d7fab8fd96a285e4ff2891 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 31 Oct 2019 09:50:00 +0100 Subject: [PATCH 03/10] cleanup --- qa/mixed-cluster/build.gradle | 1 - server/build.gradle | 1 - .../elasticsearch/cluster/metadata/MetaDataMappingService.java | 1 - .../elasticsearch/discovery/zen/PublishClusterStateAction.java | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 9e3227de57dcf..5f3690c293d85 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -75,7 +75,6 @@ for (Version bwcVersion : bwcVersions.wireCompatible) { println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",") }" println "Upgrading another node to create a mixed cluster" testClusters."${baseName}".nextNodeToNextVersion() - println "${-> testClusters."${baseName}".allHttpSocketURI[6]}" nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",") }") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName() }") diff --git a/server/build.gradle b/server/build.gradle index 1eeaa89df8521..7177ef955549b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -98,7 +98,6 @@ dependencies { compile "org.apache.lucene:lucene-spatial-extras:${versions.lucene}" compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}" compile "org.apache.lucene:lucene-suggest:${versions.lucene}" - compile "commons-lang:commons-lang:2.6" // utilities compile project(":libs:elasticsearch-cli") diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 2f337406f2679..4dc38c97a8e38 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -154,7 +154,6 @@ ClusterState executeRefresh(final ClusterState currentState, final List Date: Thu, 31 Oct 2019 11:32:02 +0100 Subject: [PATCH 04/10] failing one test --- .../search/180_locale_dependent_mapping.yml | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index d841229f306c4..080f7cc185ba3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -1,7 +1,8 @@ --- "Test Index and Search locale dependent mappings / dates": - skip: - version: "all" + version: " - 6.1.99" + reason: JDK9 only supports this with a special sysproperty added in 6.2.0 - do: indices.create: index: test_index @@ -12,27 +13,4 @@ properties: date_field: type: date - format: "8E, d MMM yyyy HH:mm:ss Z" - locale: "de" - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test_index", "_id": "1"}}' - - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' - - '{"index": {"_index": "test_index", "_id": "2"}}' - - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' - - - do: - search: - rest_total_hits_as_int: true - index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}} - - match: { hits.total: 1 } - - - do: - search: - rest_total_hits_as_int: true - index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}} - - match: { hits.total: 2 } + format: "8E, d MMM uuuu HH:mm:ss Z" From d1c169b0cf3022f5821561593e2d253405599d2c Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 31 Oct 2019 11:37:21 +0100 Subject: [PATCH 05/10] cleanup --- .../java/org/elasticsearch/ingest/common/DateFormat.java | 5 ----- server/build.gradle | 2 +- .../java/org/elasticsearch/index/mapper/MapperService.java | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 8ff1b4664fc67..e5902e7ccc01d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -82,11 +82,6 @@ private long parseMillis(String date) { @Override Function getFunction(String format, ZoneId zoneId, Locale locale) { - // support the 6.x BWC compatible way of parsing java 8 dates //TODO this is not needed?? -// if (format.startsWith("8")) { -// format = format.substring(1); -// } - boolean isUtc = ZoneOffset.UTC.equals(zoneId); DateFormatter dateFormatter = DateFormatter.forPattern(format) diff --git a/server/build.gradle b/server/build.gradle index 7177ef955549b..551b148456765 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -98,7 +98,7 @@ dependencies { compile "org.apache.lucene:lucene-spatial-extras:${versions.lucene}" compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}" compile "org.apache.lucene:lucene-suggest:${versions.lucene}" - + // utilities compile project(":libs:elasticsearch-cli") compile 'com.carrotsearch:hppc:0.8.1' diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index aea9840279de8..57d8f69709001 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -347,8 +347,6 @@ private synchronized Map internalMerge(IndexMetaData ind if (onlyUpdateIfNeeded) { DocumentMapper existingMapper = documentMapper(mappingMetaData.type()); if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) { - logger.info("he "+mappingMetaData.source()); - logger.info("he2 "+(existingMapper== null || existingMapper.mappingSource() == null ? "null" : existingMapper.mappingSource())); map.put(mappingMetaData.type(), mappingMetaData.source()); } } else { From 172b837dceeac93c7c941a715428f01a4e52826b Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 31 Oct 2019 16:56:02 +0100 Subject: [PATCH 06/10] add rest of the test --- .../search/180_locale_dependent_mapping.yml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 080f7cc185ba3..e9ba863675dfa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -13,4 +13,27 @@ properties: date_field: type: date - format: "8E, d MMM uuuu HH:mm:ss Z" + format: "E, d MMM yyyy HH:mm:ss Z" + locale: "de" + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_id": "1"}}' + - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' + - '{"index": {"_index": "test_index", "_id": "2"}}' + - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}} + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}} + - match: { hits.total: 2 } From a8be44ba637eff19a895db49436d0e6877bf1ac6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 4 Nov 2019 12:06:35 +0100 Subject: [PATCH 07/10] temp --- .../search/180_locale_dependent_mapping.yml | 25 +------------------ .../index/mapper/MapperService.java | 6 ++++- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index e9ba863675dfa..080f7cc185ba3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -13,27 +13,4 @@ properties: date_field: type: date - format: "E, d MMM yyyy HH:mm:ss Z" - locale: "de" - - do: - bulk: - refresh: true - body: - - '{"index": {"_index": "test_index", "_id": "1"}}' - - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' - - '{"index": {"_index": "test_index", "_id": "2"}}' - - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' - - - do: - search: - rest_total_hits_as_int: true - index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}} - - match: { hits.total: 1 } - - - do: - search: - rest_total_hits_as_int: true - index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}} - - match: { hits.total: 2 } + format: "8E, d MMM uuuu HH:mm:ss Z" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 57d8f69709001..e39c6722a0012 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -269,7 +269,9 @@ private void assertMappingVersion( && currentIndexMetaData.getCreationVersion().onOrAfter(Version.V_6_5_0)) { if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) { // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same - assert updatedEntries.isEmpty() : updatedEntries; +// assert updatedEntries.isEmpty() : updatedEntries; + if(!updatedEntries.isEmpty()) + logger.info("updated entries not empty" + updatedEntries); MappingMetaData defaultMapping = newIndexMetaData.defaultMapping(); if (defaultMapping != null) { @@ -347,6 +349,8 @@ private synchronized Map internalMerge(IndexMetaData ind if (onlyUpdateIfNeeded) { DocumentMapper existingMapper = documentMapper(mappingMetaData.type()); if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) { + logger.info("he "+mappingMetaData.source()); + logger.info("he2 "+(existingMapper== null || existingMapper.mappingSource() == null ? "null" : existingMapper.mappingSource())); map.put(mappingMetaData.type(), mappingMetaData.source()); } } else { From 6111036352f3842f727c550c86bb203e84946e77 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 4 Nov 2019 17:11:54 +0100 Subject: [PATCH 08/10] Actual fix --- .../main/java/org/elasticsearch/common/time/DateFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java index 1f6b9bf72c322..991dffea72085 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java @@ -141,7 +141,7 @@ static DateFormatter forPattern(String input) { .map(DateFormatters::forPattern) .collect(Collectors.toList()); - return JavaDateFormatter.combined(format, formatters); + return JavaDateFormatter.combined(input, formatters); } static String strip8Prefix(String input) { From 0c35c5549ff3097daeba379fd17874383c72ecd8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 4 Nov 2019 19:40:41 +0100 Subject: [PATCH 09/10] reomve unused logging --- .../java/org/elasticsearch/index/mapper/MapperService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index e39c6722a0012..57d8f69709001 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -269,9 +269,7 @@ private void assertMappingVersion( && currentIndexMetaData.getCreationVersion().onOrAfter(Version.V_6_5_0)) { if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) { // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same -// assert updatedEntries.isEmpty() : updatedEntries; - if(!updatedEntries.isEmpty()) - logger.info("updated entries not empty" + updatedEntries); + assert updatedEntries.isEmpty() : updatedEntries; MappingMetaData defaultMapping = newIndexMetaData.defaultMapping(); if (defaultMapping != null) { @@ -349,8 +347,6 @@ private synchronized Map internalMerge(IndexMetaData ind if (onlyUpdateIfNeeded) { DocumentMapper existingMapper = documentMapper(mappingMetaData.type()); if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) { - logger.info("he "+mappingMetaData.source()); - logger.info("he2 "+(existingMapper== null || existingMapper.mappingSource() == null ? "null" : existingMapper.mappingSource())); map.put(mappingMetaData.type(), mappingMetaData.source()); } } else { From f179172fc235a7ecf366ff922c27798c9d411ca1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 4 Nov 2019 21:52:37 +0100 Subject: [PATCH 10/10] adding the rest of the test --- .../search/180_locale_dependent_mapping.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 080f7cc185ba3..ba059e10fb66f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -14,3 +14,26 @@ date_field: type: date format: "8E, d MMM uuuu HH:mm:ss Z" + locale: "de" + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_id": "1"}}' + - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' + - '{"index": {"_index": "test_index", "_id": "2"}}' + - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}} + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}} + - match: { hits.total: 2 }