From 5dfe1935345da3799931fd4a3ebe0b6aa9c17f57 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 25 Jan 2019 16:42:29 -0700 Subject: [PATCH] Add ECS schema for user-agent ingest processor (#37727) This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs). So rather than something like this: ``` { "patch" : "3538", "major" : "70", "minor" : "0", "os" : "Mac OS X 10.14.1", "os_minor" : "14", "os_major" : "10", "name" : "Chrome", "os_name" : "Mac OS X", "device" : "Other" } ``` The structure is now like this: ``` { "name" : "Chrome", "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "os" : { "name" : "Mac OS X", "version" : "10.14.1", "full" : "Mac OS X 10.14.1" }, "device" : "Other", "version" : "70.0.3538.102" } ``` This new can be configured by setting `"ecs": true` in the processor configuration, and will be the default for 7.0. Leaving `ecs` unset or set as `false` is deprecated. Resolves #37329 This PR is against the 6.x branch and will be forward-ported with the deprecated parts removed (I will open a subsequent PR for that). --- .../ingest/processors/user-agent.asciidoc | 18 +- .../ingest/useragent/UserAgentProcessor.java | 248 +++++++++++++----- .../UserAgentProcessorFactoryTests.java | 25 +- .../useragent/UserAgentProcessorTests.java | 54 ++-- .../20_useragent_processor.yml | 67 ++++- .../test/ingest-useragent/30_custom_regex.yml | 10 +- 6 files changed, 294 insertions(+), 128 deletions(-) diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index 201e3beab8313..7c13fb5203866 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab | `regex_file` | no | - | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below). | `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `ecs` | no | `false` | Whether to return the output in Elastic Common Schema format. NOTE: ECS format will be the default in Elasticsearch 7.0 and non-ECS format is deprecated. |====== Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field: @@ -31,7 +32,8 @@ PUT _ingest/pipeline/user_agent "processors" : [ { "user_agent" : { - "field" : "agent" + "field" : "agent", + "ecs" : true } } ] @@ -60,13 +62,13 @@ Which returns "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "user_agent": { "name": "Chrome", - "major": "51", - "minor": "0", - "patch": "2704", - "os_name": "Mac OS X", - "os": "Mac OS X 10.10.5", - "os_major": "10", - "os_minor": "10", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "version": "51.0.2704", + "os": { + "name": "Mac OS X", + "version": "10.10.5", + "full": "Mac OS X 10.10.5" + }, "device": "Other" } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 6e7f588f0bd8a..f3d45ec389c0c 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -19,15 +19,19 @@ package org.elasticsearch.ingest.useragent; +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.useragent.UserAgentParser.Details; import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -40,6 +44,8 @@ public class UserAgentProcessor extends AbstractProcessor { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class)); + public static final String TYPE = "user_agent"; private final String field; @@ -47,15 +53,17 @@ public class UserAgentProcessor extends AbstractProcessor { private final Set properties; private final UserAgentParser parser; private final boolean ignoreMissing; + private final boolean useECS; public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties, - boolean ignoreMissing) { + boolean ignoreMissing, boolean useECS) { super(tag); this.field = field; this.targetField = targetField; this.parser = parser; this.properties = properties; this.ignoreMissing = ignoreMissing; + this.useECS = useECS; } boolean isIgnoreMissing() { @@ -63,7 +71,7 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) { String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (userAgent == null && ignoreMissing) { @@ -75,71 +83,134 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { Details uaClient = parser.parse(userAgent); Map uaDetails = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case NAME: - if (uaClient.userAgent != null && uaClient.userAgent.name != null) { - uaDetails.put("name", uaClient.userAgent.name); - } - else { - uaDetails.put("name", "Other"); - } - break; - case MAJOR: - if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - uaDetails.put("major", uaClient.userAgent.major); - } - break; - case MINOR: - if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { - uaDetails.put("minor", uaClient.userAgent.minor); - } - break; - case PATCH: - if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { - uaDetails.put("patch", uaClient.userAgent.patch); - } - break; - case BUILD: - if (uaClient.userAgent != null && uaClient.userAgent.build != null) { - uaDetails.put("build", uaClient.userAgent.build); - } - break; - case OS: - if (uaClient.operatingSystem != null) { - uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); - } - else { - uaDetails.put("os", "Other"); - } - break; - case OS_NAME: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { - uaDetails.put("os_name", uaClient.operatingSystem.name); - } - else { - uaDetails.put("os_name", "Other"); - } - break; - case OS_MAJOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { - uaDetails.put("os_major", uaClient.operatingSystem.major); - } - break; - case OS_MINOR: - if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { - uaDetails.put("os_minor", uaClient.operatingSystem.minor); - } - break; - case DEVICE: - if (uaClient.device != null && uaClient.device.name != null) { - uaDetails.put("device", uaClient.device.name); - } - else { - uaDetails.put("device", "Other"); - } - break; + if (useECS) { + // Parse the user agent in the ECS (Elastic Common Schema) format + for (Property property : this.properties) { + switch (property) { + case ORIGINAL: + uaDetails.put("original", userAgent); + break; + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case VERSION: + StringBuilder version = new StringBuilder(); + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + version.append(uaClient.userAgent.major); + if (uaClient.userAgent.minor != null) { + version.append(".").append(uaClient.userAgent.minor); + if (uaClient.userAgent.patch != null) { + version.append(".").append(uaClient.userAgent.patch); + if (uaClient.userAgent.build != null) { + version.append(".").append(uaClient.userAgent.build); + } + } + } + uaDetails.put("version", version.toString()); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + Map osDetails = new HashMap<>(3); + if (uaClient.operatingSystem.name != null) { + osDetails.put("name", uaClient.operatingSystem.name); + StringBuilder sb = new StringBuilder(); + if (uaClient.operatingSystem.major != null) { + sb.append(uaClient.operatingSystem.major); + if (uaClient.operatingSystem.minor != null) { + sb.append(".").append(uaClient.operatingSystem.minor); + if (uaClient.operatingSystem.patch != null) { + sb.append(".").append(uaClient.operatingSystem.patch); + if (uaClient.operatingSystem.build != null) { + sb.append(".").append(uaClient.operatingSystem.build); + } + } + } + osDetails.put("version", sb.toString()); + osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); + } + uaDetails.put("os", osDetails); + } + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } else { + uaDetails.put("device", "Other"); + } + break; + } + } + } else { + // Deprecated format, removed in 7.0 + for (Property property : this.properties) { + switch (property) { + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case MAJOR: + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } else { + uaDetails.put("device", "Other"); + } + break; + } } } @@ -199,6 +270,10 @@ UserAgentParser getUaParser() { return parser; } + public boolean isUseECS() { + return useECS; + } + public static final class Factory implements Processor.Factory { private final Map userAgentParsers; @@ -215,6 +290,7 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", false); UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -236,17 +312,51 @@ public UserAgentProcessor create(Map factories, Strin properties = EnumSet.allOf(Property.class); } - return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing); + if (useECS == false) { + deprecationLogger.deprecated("setting [ecs] to false for non-common schema " + + "format is deprecated and will be removed in 7.0, set to true to use the non-deprecated format"); + } + + return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS); } } enum Property { - NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; + NAME, + // Deprecated in 6.7 (superceded by VERSION), to be removed in 7.0 + @Deprecated MAJOR, + @Deprecated MINOR, + @Deprecated PATCH, + OS, + // Deprecated in 6.7 (superceded by just using OS), to be removed in 7.0 + @Deprecated OS_NAME, + @Deprecated OS_MAJOR, + @Deprecated OS_MINOR, + DEVICE, + @Deprecated BUILD, // Same deprecated as OS_* above + ORIGINAL, + VERSION; + + private static Set DEPRECATED_PROPERTIES; + + static { + Set deprecated = new HashSet<>(); + for (Field field : Property.class.getFields()) { + if (field.isEnumConstant() && field.isAnnotationPresent(Deprecated.class)) { + deprecated.add(valueOf(field.getName())); + } + } + DEPRECATED_PROPERTIES = deprecated; + } public static Property parseProperty(String propertyName) { try { - return valueOf(propertyName.toUpperCase(Locale.ROOT)); + Property value = valueOf(propertyName.toUpperCase(Locale.ROOT)); + if (DEPRECATED_PROPERTIES.contains(value)) { + deprecationLogger.deprecated("the [{}] property is deprecated for the user-agent processor", propertyName); + } + return value; } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index d9c6fc17620da..1aee1c0ed1dd7 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.ingest.useragent; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -27,17 +28,21 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStreamReader; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -78,6 +83,7 @@ public void testBuildDefaults() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); + config.put("ecs", true); String processorTag = randomAlphaOfLength(10); @@ -90,6 +96,7 @@ public void testBuildDefaults() throws Exception { assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); assertFalse(processor.isIgnoreMissing()); + assertTrue(processor.isUseECS()); } public void testBuildWithIgnoreMissing() throws Exception { @@ -98,6 +105,7 @@ public void testBuildWithIgnoreMissing() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("ignore_missing", true); + config.put("ecs", true); String processorTag = randomAlphaOfLength(10); @@ -118,6 +126,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_target_field"); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -130,6 +139,7 @@ public void testBuildRegexFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("regex_file", regexWithoutDevicesFilename); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -155,8 +165,17 @@ public void testBuildFields() throws Exception { Set properties = EnumSet.noneOf(UserAgentProcessor.Property.class); List fieldNames = new ArrayList<>(); int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length); + Set warnings = new HashSet<>(); + Set deprecated = Arrays.stream(UserAgentProcessor.Property.class.getFields()) + .filter(Field::isEnumConstant) + .filter(field -> field.isAnnotationPresent(Deprecated.class)) + .map(field -> UserAgentProcessor.Property.valueOf(field.getName())) + .collect(Collectors.toSet()); for (int i = 0; i < numFields; i++) { UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i]; + if (deprecated.contains(property)) { + warnings.add("the [" + property.name().toLowerCase(Locale.ROOT) + "] property is deprecated for the user-agent processor"); + } properties.add(property); fieldNames.add(property.name().toLowerCase(Locale.ROOT)); } @@ -164,10 +183,14 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); + if (warnings.size() > 0) { + assertWarnings(warnings.toArray(Strings.EMPTY_ARRAY)); + } } public void testInvalidProperty() throws Exception { @@ -179,7 +202,7 @@ public void testInvalidProperty() throws Exception { ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " - + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD, ORIGINAL, VERSION]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 0a8b453724c90..59aa68be6d31d 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -48,12 +48,12 @@ public static void setupProcessor() throws IOException { UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); } public void testNullValueWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -63,7 +63,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -72,7 +72,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -82,7 +82,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); @@ -103,16 +103,13 @@ public void testCommonBrowser() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Chrome")); - assertThat(target.get("major"), is("33")); - assertThat(target.get("minor"), is("0")); - assertThat(target.get("patch"), is("1750")); - assertNull(target.get("build")); - - assertThat(target.get("os"), is("Mac OS X 10.9.2")); - assertThat(target.get("os_name"), is("Mac OS X")); - assertThat(target.get("os_major"), is("10")); - assertThat(target.get("os_minor"), is("9")); + assertThat(target.get("version"), is("33.0.1750")); + Map os = new HashMap<>(); + os.put("name", "Mac OS X"); + os.put("version", "10.9.2"); + os.put("full", "Mac OS X 10.9.2"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Other")); } @@ -131,15 +128,13 @@ public void testUncommonDevice() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Android")); - assertThat(target.get("major"), is("3")); - assertThat(target.get("minor"), is("0")); - assertNull(target.get("patch")); - assertNull(target.get("build")); + assertThat(target.get("version"), is("3.0")); - assertThat(target.get("os"), is("Android 3.0")); - assertThat(target.get("os_name"), is("Android")); - assertThat(target.get("os_major"), is("3")); - assertThat(target.get("os_minor"), is("0")); + Map os = new HashMap<>(); + os.put("name", "Android"); + os.put("version", "3.0"); + os.put("full", "Android 3.0"); + assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Motorola Xoom")); } @@ -158,15 +153,9 @@ public void testSpider() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("EasouSpider")); - assertNull(target.get("major")); - assertNull(target.get("minor")); - assertNull(target.get("patch")); - assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("version")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Spider")); } @@ -190,10 +179,7 @@ public void testUnknown() throws Exception { assertNull(target.get("patch")); assertNull(target.get("build")); - assertThat(target.get("os"), is("Other")); - assertThat(target.get("os_name"), is("Other")); - assertNull(target.get("os_major")); - assertNull(target.get("os_minor")); + assertNull(target.get("os")); assertThat(target.get("device"), is("Other")); } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 0964e69a99b6d..02b7ca4ce06e1 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -9,7 +9,8 @@ "processors": [ { "user_agent" : { - "field" : "field1" + "field" : "field1", + "ecs": true } } ] @@ -31,13 +32,9 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } - - match: { _source.user_agent.os: "Mac OS X 10.9.2" } - - match: { _source.user_agent.os_name: "Mac OS X" } - - match: { _source.user_agent.os_major: "10" } - - match: { _source.user_agent.os_minor: "9" } - - match: { _source.user_agent.major: "33" } - - match: { _source.user_agent.minor: "0" } - - match: { _source.user_agent.patch: "1750" } + - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } + - match: { _source.user_agent.version: "33.0.1750" } - match: { _source.user_agent.device: "Other" } --- @@ -52,6 +49,7 @@ { "user_agent" : { "field" : "field1", + "ecs": true, "target_field": "field2", "properties": ["os"] } @@ -74,7 +72,7 @@ type: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os.full: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name - is_false: _source.field2.os_name @@ -84,3 +82,54 @@ - is_false: _source.field2.minor - is_false: _source.field2.patch - is_false: _source.field2.device + +--- +"Test user agent processor with non-ECS schema": + - skip: + features: warnings + + - do: + warnings: + - "setting [ecs] to false for non-common schema format is deprecated and will be removed in 7.0, set to true to use the non-deprecated format" + - "the [os_major] property is deprecated for the user-agent processor" + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "user_agent" : { + "field" : "field1", + "target_field": "field2", + "properties": ["os", "os_major"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os_major: "10" } + - is_false: _source.user_agent + - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch + - is_false: _source.field2.device diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 5a6cd8f7a86f6..e034fcd7b3867 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -10,6 +10,7 @@ { "user_agent" : { "field": "field1", + "ecs": true, "regex_file": "test-regexes.yml" } } @@ -32,11 +33,6 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } - - match: { _source.user_agent.os: "Other" } - - match: { _source.user_agent.os_name: "Other" } - match: { _source.user_agent.device: "Other" } - - is_false: _source.user_agent.os_major - - is_false: _source.user_agent.os_minor - - is_false: _source.user_agent.major - - is_false: _source.user_agent.minor - - is_false: _source.user_agent.patch + - is_false: _source.user_agent.os + - is_false: _source.user_agent.version