From b0567d7c1894306690fb3aea8b8b2ebc07f9f441 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 19 Aug 2015 20:38:00 +0200 Subject: [PATCH] [build] Update to elasticsearch 2.0.0 Fix compiling issues. Also fix test issues. Need now to move old integration tests to RESTIT Closes #148. --- pom.xml | 12 ++++++++-- .../mapper/attachment/AttachmentMapper.java | 15 ++++-------- .../attachment/test/MapperTestUtils.java | 23 +++++-------------- .../AttachmentIntegrationTestCase.java | 6 ++--- .../EncryptedAttachmentIntegrationTests.java | 11 ++++----- .../SimpleAttachmentIntegrationTests.java | 17 +++++++------- .../test/standalone/StandaloneRunner.java | 4 ++-- .../test/unit/AttachmentUnitTestCase.java | 4 ++-- 8 files changed, 41 insertions(+), 51 deletions(-) diff --git a/pom.xml b/pom.xml index d941152c6bcb0..5271860fa3cee 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,6 @@ org.elasticsearch elasticsearch-mapper-attachments 3.0.0-SNAPSHOT - jar Elasticsearch Mapper Attachment plugin The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika. https://github.com/elastic/elasticsearch-mapper-attachments/ @@ -26,7 +25,7 @@ - org.elasticsearch + org.elasticsearch.plugin elasticsearch-plugin 2.0.0-SNAPSHOT @@ -34,9 +33,18 @@ 2.0.0-SNAPSHOT + org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin + + org.elasticsearch.distribution.fully-loaded + elasticsearch + ${elasticsearch.version} + pom + test + + org.apache.tika diff --git a/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java b/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java index 3e1b5a0dcf143..e034f4caa67d8 100644 --- a/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java @@ -20,15 +20,12 @@ package org.elasticsearch.index.mapper.attachment; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.Constants; import org.apache.tika.Tika; import org.apache.tika.language.LanguageIdentifier; import org.apache.tika.metadata.Metadata; import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; @@ -36,9 +33,7 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.*; -import org.elasticsearch.index.mapper.core.AbstractFieldMapper; import java.io.IOException; import java.util.*; @@ -67,7 +62,7 @@ * _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for * tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues. */ -public class AttachmentMapper extends AbstractFieldMapper { +public class AttachmentMapper extends FieldMapper { private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment"); @@ -115,7 +110,7 @@ public String value(Object value) { } } - public static class Builder extends AbstractFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private ContentPath.Type pathType = Defaults.PATH_TYPE; @@ -207,10 +202,10 @@ public AttachmentMapper build(BuilderContext context) { FieldMapper contentMapper; if (context.indexCreatedVersion().before(Version.V_2_0_0)) { // old behavior, we need the content to be indexed under the attachment field name - if (contentBuilder instanceof AbstractFieldMapper.Builder == false) { + if (contentBuilder instanceof FieldMapper.Builder == false) { throw new IllegalStateException("content field for attachment must be a field mapper"); } - ((AbstractFieldMapper.Builder)contentBuilder).indexName(name); + ((FieldMapper.Builder)contentBuilder).indexName(name); contentBuilder.name = name + "." + FieldNames.CONTENT; contentMapper = (FieldMapper) contentBuilder.build(context); context.path().add(name); @@ -329,7 +324,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext Map propNode = (Map) entry1.getValue(); Mapper.Builder mapperBuilder = findMapperBuilder(propNode, propName, parserContext); - if (parseMultiField((AbstractFieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) { + if (parseMultiField((FieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) { fieldsIterator.remove(); } else if (propName.equals(name) && parserContext.indexVersionCreated().before(Version.V_2_0_0)) { builder.content(mapperBuilder); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java index 5b7c62bfcf6e0..573c1f35e661f 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java @@ -32,17 +32,12 @@ import org.elasticsearch.index.IndexNameModule; import org.elasticsearch.index.analysis.AnalysisModule; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.indices.analysis.IndicesAnalysisModule; import org.elasticsearch.indices.analysis.IndicesAnalysisService; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; -import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; -import org.elasticsearch.threadpool.ThreadPool; import java.nio.file.Path; import java.util.Locale; @@ -52,23 +47,17 @@ public class MapperTestUtils { - public static MapperService newMapperService(Path tempDir, ThreadPool testingThreadPool) { + public static MapperService newMapperService(Path tempDir) { return newMapperService(new Index("test"), Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put("path.home", tempDir) - .build(), testingThreadPool); + .build()); } - public static MapperService newMapperService(Index index, Settings indexSettings, ThreadPool testingThreadPool) { - NoneCircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); - return new MapperService(index, + public static MapperService newMapperService(Index index, Settings indexSettings) { + return new MapperService(index, indexSettings, newAnalysisService(indexSettings), - new IndexFieldDataService(index, Settings.Builder.EMPTY_SETTINGS, - new IndicesFieldDataCache(Settings.Builder.EMPTY_SETTINGS, - new IndicesFieldDataCacheListener(circuitBreakerService), - testingThreadPool), - circuitBreakerService), newSimilarityLookupService(indexSettings), null); } @@ -106,8 +95,8 @@ public static DocumentMapperParser newMapperParser(Settings settings) { .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(settings) .build(); - MapperService mapperService = new MapperService(new Index("test"), forcedSettings, newAnalysisService(forcedSettings), null, newSimilarityLookupService(forcedSettings), null); - return new DocumentMapperParser(new Index("test"), forcedSettings, mapperService, MapperTestUtils.newAnalysisService(forcedSettings), null, null); + MapperService mapperService = new MapperService(new Index("test"), forcedSettings, newAnalysisService(forcedSettings), newSimilarityLookupService(forcedSettings), null); + return new DocumentMapperParser(forcedSettings, mapperService, MapperTestUtils.newAnalysisService(forcedSettings), null, null); } /** diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/AttachmentIntegrationTestCase.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/AttachmentIntegrationTestCase.java index d6bbc83fa3df7..d3e01e4b53dcc 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/AttachmentIntegrationTestCase.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/AttachmentIntegrationTestCase.java @@ -19,15 +19,15 @@ package org.elasticsearch.index.mapper.attachment.test.integration; -import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matcher; import org.junit.BeforeClass; import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isLocaleCompatible; import static org.hamcrest.Matchers.not; -@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE) -public class AttachmentIntegrationTestCase extends ElasticsearchIntegrationTest { +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) +public class AttachmentIntegrationTestCase extends ESIntegTestCase { protected static boolean expectError; diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/EncryptedAttachmentIntegrationTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/EncryptedAttachmentIntegrationTests.java index 6a1f94d613d35..d5415ced36c76 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/EncryptedAttachmentIntegrationTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/EncryptedAttachmentIntegrationTests.java @@ -22,22 +22,21 @@ import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin; import org.junit.Test; -import org.apache.lucene.util.LuceneTestCase.Slow; import static org.elasticsearch.client.Requests.putMappingRequest; -import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; -import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.equalTo; /** * Test case for issue https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18 + * TODO Change this to a real IT */ -@Slow public class EncryptedAttachmentIntegrationTests extends AttachmentIntegrationTestCase { private boolean ignore_errors = true; @@ -45,7 +44,7 @@ public class EncryptedAttachmentIntegrationTests extends AttachmentIntegrationTe protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", MapperAttachmentsPlugin.class.getName()) .build(); } diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/SimpleAttachmentIntegrationTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/SimpleAttachmentIntegrationTests.java index e58ce74a627f1..61649e4e66166 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/SimpleAttachmentIntegrationTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/SimpleAttachmentIntegrationTests.java @@ -24,32 +24,29 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin; import org.elasticsearch.search.highlight.HighlightField; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.apache.lucene.util.LuceneTestCase.Slow; import static org.elasticsearch.client.Requests.putMappingRequest; -import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; -import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.*; /** - * + * TODO Change this to real IT */ -@Slow public class SimpleAttachmentIntegrationTests extends AttachmentIntegrationTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", MapperAttachmentsPlugin.class.getName()) .build(); } @@ -164,7 +161,8 @@ public void testContentTypeAndName() throws Exception { * As for now, we don't support a global `copy_to` property for `attachment` type. * So this test is failing. */ - @Test @Ignore + /* + @Test public void testCopyTo() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/integration/simple/copy-to.json"); byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt"); @@ -180,6 +178,7 @@ public void testCopyTo() throws Exception { countResponse = client().prepareCount("test").setQuery(queryStringQuery("Queen").defaultField("copy")).execute().get(); assertThatWithError(countResponse.getCount(), equalTo(1l)); } + */ @Test public void testHighlightAttachment() throws Exception { diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java index afa5b563575de..c3ff9d4045c5c 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java @@ -45,8 +45,8 @@ import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; import static org.elasticsearch.common.io.Streams.copy; -import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; /** * This class provides a simple main class which can be used to test what is extracted from a given binary file. @@ -109,7 +109,7 @@ public ExitStatus execute(Settings settings, Environment env) throws Exception { BytesReference json = builder.endObject().endObject().bytes(); - ParseContext.Document doc = docMapper.parse("person", "1", json).rootDoc(); + ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); terminal.println("## Extracted text"); terminal.println("--------------------- BEGIN -----------------------"); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/AttachmentUnitTestCase.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/AttachmentUnitTestCase.java index 4884427f61362..dfa96a75ce938 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/AttachmentUnitTestCase.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/AttachmentUnitTestCase.java @@ -22,13 +22,13 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ElasticsearchTestCase; +import org.elasticsearch.test.ESTestCase; import org.junit.Before; import org.junit.BeforeClass; import static org.elasticsearch.index.mapper.attachment.test.MapperTestUtils.assumeCorrectLocale; -public class AttachmentUnitTestCase extends ElasticsearchTestCase { +public class AttachmentUnitTestCase extends ESTestCase { /** * We can have issues with some JVMs and Locale * See https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/105