URI parts ingest processor

x-pack/plugin/ingest/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'elasticsearch.internal-cluster-test'
+esplugin {
+  name 'x-pack-ingest'
+  description 'Elasticsearch Expanded Pack Plugin - Ingest'
+  classname 'org.elasticsearch.xpack.ingest.IngestPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-ingest'
+
+dependencies {
+  compileOnly project(path: xpackModule('core'), configuration: 'default')
+  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testImplementation project(path: ':modules:ingest-common')
+  testImplementation project(path: ':modules:lang-mustache')
+  testImplementation project(path: ':modules:geo')
+  testImplementation project(path: xpackModule('monitoring'), configuration: 'testArtifacts')
+}
+
+addQaCheckDependencies()

x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ingest;
+
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Map;
+
+public class IngestPlugin extends Plugin implements org.elasticsearch.plugins.IngestPlugin {
+
+    @Override
+    public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+        return Map.of(UrlPartsProcessor.TYPE, new UrlPartsProcessor.Factory());
+    }
+}

x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/UrlPartsProcessor.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ingest;
+
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+public class UrlPartsProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "url_parts";
+
+    private final String field;
+    private final String targetField;
+    private final boolean removeIfSuccessful;
+    private final boolean keepOriginal;
+
+    UrlPartsProcessor(String tag, String description, String field, String targetField, boolean removeIfSuccessful, boolean keepOriginal) {
+        super(tag, description);
+        this.field = field;
+        this.targetField = targetField;
+        this.removeIfSuccessful = removeIfSuccessful;
+        this.keepOriginal = keepOriginal;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public String getTargetField() {
+        return targetField;
+    }
+
+    public boolean getRemoveIfSuccessful() {
+        return removeIfSuccessful;
+    }
+
+    public boolean getKeepOriginal() {
+        return keepOriginal;
+    }
+
+    @Override
+    public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
+        String value = ingestDocument.getFieldValue(field, String.class);
+
+        URL url;
+        try {
+            url = new URL(value);
+        } catch (MalformedURLException e) {
+            throw new IllegalArgumentException("unable to parse URL [" + value + "]");
+        }
+        var urlParts = new HashMap<String, Object>();
+        urlParts.put("domain", url.getHost());
+        if (url.getRef() != null) {
+            urlParts.put("fragment", url.getRef());
+        }
+        if (keepOriginal) {
+            urlParts.put("original", value);
+        }
+        final String path = url.toURI().getPath();
+        if (path != null) {
+            urlParts.put("path", path);
+            if (path.contains(".")) {
+                int periodIndex = path.lastIndexOf('.');
+                urlParts.put("extension", periodIndex < path.length() ? path.substring(periodIndex + 1) : "");
+            }
+        }
+        if (url.getPort() != -1) {
+            urlParts.put("port", url.getPort());
+        }
+        if (url.getQuery() != null) {
+            urlParts.put("query", url.getQuery());
+        }
+        urlParts.put("scheme", url.getProtocol());
+        final String userInfo = url.getUserInfo();
+        if (userInfo != null) {
+            urlParts.put("user_info", userInfo);
+            if (userInfo.contains(":")) {
+                int colonIndex = userInfo.indexOf(":");
+                urlParts.put("username", userInfo.substring(0, colonIndex));
+                urlParts.put("password", colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : "");
+            }
+        }
+
+        if (removeIfSuccessful && targetField.equals(field) == false) {
+            ingestDocument.removeField(field);
+        }
+        ingestDocument.setFieldValue(targetField, urlParts);
+        return ingestDocument;
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static final class Factory implements Processor.Factory {
+
+        @Override
+        public UrlPartsProcessor create(
+            Map<String, Processor.Factory> registry,
+            String processorTag,
+            String description,
+            Map<String, Object> config
+        ) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url");
+            boolean removeIfSuccessful = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "remove_if_successful", false);
+            boolean keepOriginal = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "keep_original", true);
+            return new UrlPartsProcessor(processorTag, description, field, targetField, removeIfSuccessful, keepOriginal);
+        }
+    }
+}

x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UrlPartsProcessorFactoryTests.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ingest;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class UrlPartsProcessorFactoryTests extends ESTestCase {
+
+    private UrlPartsProcessor.Factory factory;
+
+    @Before
+    public void init() {
+        factory = new UrlPartsProcessor.Factory();
+    }
+
+    public void testCreate() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        String field = randomAlphaOfLength(6);
+        config.put("field", field);
+        String targetField = "url";
+        if (randomBoolean()) {
+            targetField = randomAlphaOfLength(6);
+            config.put("target_field", targetField);
+        }
+        boolean removeIfSuccessful = randomBoolean();
+        config.put("remove_if_successful", removeIfSuccessful);
+        boolean keepOriginal = randomBoolean();
+        config.put("keep_original", keepOriginal);
+
+        String processorTag = randomAlphaOfLength(10);
+        UrlPartsProcessor urlPartsProcessor = factory.create(null, processorTag, null, config);
+        assertThat(urlPartsProcessor.getTag(), equalTo(processorTag));
+        assertThat(urlPartsProcessor.getField(), equalTo(field));
+        assertThat(urlPartsProcessor.getTargetField(), equalTo(targetField));
+        assertThat(urlPartsProcessor.getRemoveIfSuccessful(), equalTo(removeIfSuccessful));
+        assertThat(urlPartsProcessor.getKeepOriginal(), equalTo(keepOriginal));
+    }
+
+    public void testCreateNoFieldPresent() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("value", "value1");
+        try {
+            factory.create(null, null, null, config);
+            fail("factory create should have failed");
+        } catch (ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
+        }
+    }
+
+    public void testCreateNullField() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", null);
+        try {
+            factory.create(null, null, null, config);
+            fail("factory create should have failed");
+        } catch (ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
+        }
+    }
+}