diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java index fa46540e29f7a..3bbb0539f193a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -236,7 +237,7 @@ boolean processDatabase(String id, DatabaseConfiguration database) throws IOExce logger.debug("Processing database [{}] for configuration [{}]", name, database.id()); try (ProviderDownload downloader = downloaderFor(database)) { - if (downloader.validCredentials()) { + if (downloader != null && downloader.validCredentials()) { // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, // but the downloading and indexing of database code expects it to be there, so we add it on here before continuing final String fileName = name + ".mmdb"; @@ -443,10 +444,17 @@ private void scheduleNextRun(TimeValue time) { } } + @Nullable private ProviderDownload downloaderFor(DatabaseConfiguration database) { - assert database.provider() instanceof DatabaseConfiguration.Maxmind - : "Attempt to use maxmind downloader with a provider of type" + database.provider().getClass(); - return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider()); + if (database.provider() instanceof DatabaseConfiguration.Maxmind) { + return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider()); + } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo) { + // as a temporary implementation detail, null here means 'not actually supported *just yet*' + return null; + } else { + assert false : "Attempted to use database downloader with unsupported provider type [" + database.provider().getClass() + "]"; + return null; + } } class MaxmindDownload implements ProviderDownload { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index a7828a9f3a0b7..eacf2e5a2ee57 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -55,6 +55,7 @@ import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.Factory.downloadDatabaseOnPipelineCreation; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE; /** * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node. @@ -297,9 +298,18 @@ private static boolean hasAtLeastOneGeoipProcessor(Map processor return false; } - final Map processorConfig = (Map) processor.get(GEOIP_TYPE); - if (processorConfig != null) { - return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + { + final Map processorConfig = (Map) processor.get(GEOIP_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + } + } + + { + final Map processorConfig = (Map) processor.get(IP_LOCATION_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(IP_LOCATION_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + } } return isProcessorWithOnFailureGeoIpProcessor(processor, downloadDatabaseOnPipelineCreation) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index f8ca6d87924a4..6c64cb755bb32 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -42,6 +42,7 @@ public final class GeoIpProcessor extends AbstractProcessor { + "in a future version of Elasticsearch"; // TODO add a message about migration? public static final String GEOIP_TYPE = "geoip"; + public static final String IP_LOCATION_TYPE = "ip_location"; private final String type; private final String field; @@ -225,7 +226,7 @@ public Processor create( final Map config ) throws IOException { String ipField = readStringProperty(type, processorTag, config, "field"); - String targetField = readStringProperty(type, processorTag, config, "target_field", "geoip"); + String targetField = readStringProperty(type, processorTag, config, "target_field", type); String databaseFile = readStringProperty(type, processorTag, config, "database_file", "GeoLite2-City.mmdb"); List propertyNames = readOptionalList(type, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(type, processorTag, config, "ignore_missing", false); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 49932f342086e..cc0bec583483e 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -71,6 +71,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Map.entry; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; @@ -129,7 +130,10 @@ public Map getProcessors(Processor.Parameters paramet parameters.ingestService.getClusterService() ); databaseRegistry.set(registry); - return Map.of(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)); + return Map.ofEntries( + entry(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)), + entry(GeoIpProcessor.IP_LOCATION_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.IP_LOCATION_TYPE, registry)) + ); } @Override @@ -239,6 +243,11 @@ public List getNamedWriteables() { DatabaseConfiguration.Maxmind.NAME, DatabaseConfiguration.Maxmind::new ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), new NamedWriteableRegistry.Entry( DatabaseConfiguration.Provider.class, DatabaseConfiguration.Local.NAME, diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index 3399b71879e26..a26364f9305e1 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; @@ -78,8 +79,19 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i // "GeoLite2-Country" ); + public static final Set IPINFO_NAMES = Set.of( + // these file names are from https://ipinfo.io/developers/database-filename-reference + "asn", // "Free IP to ASN" + "country", // "Free IP to Country" + // "country_asn" // "Free IP to Country + IP to ASN", not supported at present + "standard_asn", // commercial "ASN" + "standard_location", // commercial "IP Geolocation" + "standard_privacy" // commercial "Privacy Detection" (sometimes "Anonymous IP") + ); + private static final ParseField NAME = new ParseField("name"); private static final ParseField MAXMIND = new ParseField(Maxmind.NAME); + private static final ParseField IPINFO = new ParseField(Ipinfo.NAME); private static final ParseField WEB = new ParseField(Web.NAME); private static final ParseField LOCAL = new ParseField(Local.NAME); @@ -89,12 +101,21 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i (a, id) -> { String name = (String) a[0]; Provider provider; + + // one and only one provider object must be present + final long numNonNulls = Arrays.stream(a, 1, a.length).filter(Objects::nonNull).count(); + if (numNonNulls != 1) { + throw new IllegalArgumentException("Exactly one provider object must be specified, but [" + numNonNulls + "] were found"); + } + if (a[1] != null) { provider = (Maxmind) a[1]; } else if (a[2] != null) { - provider = (Web) a[2]; + provider = (Ipinfo) a[2]; + } else if (a[3] != null) { + provider = (Web) a[3]; } else { - provider = (Local) a[3]; + provider = (Local) a[4]; } return new DatabaseConfiguration(id, name, provider); } @@ -107,6 +128,7 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i (parser, id) -> Maxmind.PARSER.apply(parser, null), MAXMIND ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Ipinfo.PARSER.apply(parser, null), IPINFO); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL); } @@ -194,8 +216,16 @@ public ActionRequestValidationException validate() { err.addValidationError("invalid name [" + name + "]: cannot be empty"); } - if (MAXMIND_NAMES.contains(name) == false) { - err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + // provider-specific name validation + if (provider instanceof Maxmind) { + if (MAXMIND_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + } + } + if (provider instanceof Ipinfo) { + if (IPINFO_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + IPINFO_NAMES + "])"); + } } // important: the name must be unique across all configurations of this same type, @@ -234,7 +264,7 @@ public String getWriteableName() { private static final ParseField ACCOUNT_ID = new ParseField("account_id"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("maxmind", false, (a, id) -> { String accountId = (String) a[0]; return new Maxmind(accountId); }); @@ -247,10 +277,6 @@ public Maxmind(StreamInput in) throws IOException { this(in.readString()); } - public static Maxmind parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(accountId); @@ -270,6 +296,37 @@ public boolean isReadOnly() { } } + public record Ipinfo() implements Provider { + public static final String NAME = "ipinfo"; + + // this'll become a ConstructingObjectParser once we accept the token (securely) in the json definition + private static final ObjectParser PARSER = new ObjectParser<>("ipinfo", Ipinfo::new); + + public Ipinfo(StreamInput in) throws IOException { + this(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return false; + } + } + public record Local(String type) implements Provider { public static final String NAME = "local"; @@ -288,10 +345,6 @@ public Local(StreamInput in) throws IOException { this(in.readString()); } - public static Local parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(type); @@ -325,10 +378,6 @@ public Web(StreamInput in) throws IOException { this(); } - public static Web parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException {} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java index e836821a3b2f2..78ea73250d632 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java @@ -27,7 +27,7 @@ public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}")); + return List.of(new Route(DELETE, "/_ingest/ip_location/database/{id}"), new Route(DELETE, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java index f34f388f22965..af446ee8d2bd9 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java @@ -26,7 +26,12 @@ public class RestGetDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}")); + return List.of( + new Route(GET, "/_ingest/ip_location/database"), + new Route(GET, "/_ingest/ip_location/database/{id}"), + new Route(GET, "/_ingest/geoip/database"), + new Route(GET, "/_ingest/geoip/database/{id}") + ); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java index c0b7a3f59f3aa..95b40df12fd1f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java @@ -29,7 +29,7 @@ public class RestPutDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(PUT, "/_ingest/geoip/database/{id}")); + return List.of(new Route(PUT, "/_ingest/ip_location/database/{id}"), new Route(PUT, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java index fda0e12bb1b76..dfb8fa78089d2 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; import org.elasticsearch.injection.guice.Inject; @@ -41,6 +42,8 @@ import java.util.Map; import java.util.Optional; +import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO; + public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class); @@ -58,6 +61,7 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) { } }; + private final FeatureService featureService; private final MasterServiceTaskQueue updateDatabaseConfigurationTaskQueue; @Inject @@ -66,7 +70,8 @@ public TransportPutDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + FeatureService featureService ) { super( PutDatabaseConfigurationAction.NAME, @@ -79,6 +84,7 @@ public TransportPutDatabaseConfigurationAction( AcknowledgedResponse::readFrom, EsExecutors.DIRECT_EXECUTOR_SERVICE ); + this.featureService = featureService; this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( "update-geoip-database-configuration-state-update", Priority.NORMAL, @@ -89,6 +95,19 @@ public TransportPutDatabaseConfigurationAction( @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { final String id = request.getDatabase().id(); + + // if this is an ipinfo configuration, then make sure the whole cluster supports that feature + if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo + && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) { + listener.onFailure( + new IllegalArgumentException( + "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature " + + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id() + ) + ); + return; + } + updateDatabaseConfigurationTaskQueue.submitTask( Strings.format("update-geoip-database-configuration-[%s]", id), new UpdateDatabaseConfigurationTask(listener, request.getDatabase()), diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java index 33356ad4235dc..76b2896afe7a0 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Ipinfo; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Local; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Web; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.Set; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.IPINFO_NAMES; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES; public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase { @@ -44,13 +46,14 @@ protected DatabaseConfiguration createTestInstance() { } public static DatabaseConfiguration randomDatabaseConfiguration(String id) { + boolean useIpinfo = randomBoolean(); DatabaseConfiguration.Provider provider = switch (between(0, 2)) { - case 0 -> new Maxmind(randomAlphaOfLength(5)); + case 0 -> useIpinfo ? new Ipinfo() : new Maxmind(randomAlphaOfLength(5)); case 1 -> new Web(); case 2 -> new Local(randomAlphaOfLength(10)); default -> throw new AssertionError("failure, got illegal switch case"); }; - return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), provider); + return new DatabaseConfiguration(id, useIpinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES), provider); } @Override @@ -61,21 +64,21 @@ protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) { case 1: return new DatabaseConfiguration( instance.id(), - randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)), + randomValueOtherThan( + instance.name(), + () -> instance.provider() instanceof Ipinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES) + ), instance.provider() ); case 2: DatabaseConfiguration.Provider provider = instance.provider(); - DatabaseConfiguration.Provider modifiedProvider; - if (provider instanceof Maxmind maxmind) { - modifiedProvider = new Maxmind(((Maxmind) instance.provider()).accountId() + randomAlphaOfLength(2)); - } else if (provider instanceof Web) { - modifiedProvider = new Maxmind(randomAlphaOfLength(20)); // can't modify a Web - } else if (provider instanceof Local local) { - modifiedProvider = new Local(local.type() + randomAlphaOfLength(2)); - } else { - throw new AssertionError("Unexpected provider type: " + provider.getClass()); - } + DatabaseConfiguration.Provider modifiedProvider = switch (provider) { + case Maxmind maxmind -> new Maxmind(maxmind.accountId() + randomAlphaOfLength(2)); + case Ipinfo ignored -> new Local(randomAlphaOfLength(20)); // can't modify Ipinfo + case Web ignored -> new Local(randomAlphaOfLength(20)); // can't modify a Web + case Local local -> new Local(local.type() + randomAlphaOfLength(2)); + default -> throw new AssertionError("Unexpected provider type: " + provider.getClass()); + }; return new DatabaseConfiguration(instance.id(), instance.name(), modifiedProvider); default: throw new AssertionError("failure, got illegal switch case"); diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index 04fd2ac6a8189..a1104505bc240 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -1,8 +1,20 @@ +--- setup: - requires: cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"] reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results" +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 + --- "Test adding, getting, and removing geoip databases": - do: diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml new file mode 100644 index 0000000000000..fd73c715a5ac5 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml @@ -0,0 +1,45 @@ +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ipinfo support added in 8.16" + +--- +"Test ip_location processor with defaults": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "ip_location" : { + "field" : "field1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + pipeline: "my_pipeline" + body: {field1: "89.160.20.128"} + + - do: + get: + index: test + id: "1" + - match: { _source.field1: "89.160.20.128" } + - length: { _source.ip_location: 7 } + - match: { _source.ip_location.city_name: "Linköping" } + - match: { _source.ip_location.country_iso_code: "SE" } + - match: { _source.ip_location.location.lon: 15.6167 } + - match: { _source.ip_location.location.lat: 58.4167 } + - match: { _source.ip_location.region_iso_code: "SE-E" } + - match: { _source.ip_location.country_name: "Sweden" } + - match: { _source.ip_location.region_name: "Östergötland County" } + - match: { _source.ip_location.continent_name: "Europe" } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml new file mode 100644 index 0000000000000..e2e9a1fdb5e28 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml @@ -0,0 +1,137 @@ +--- +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ip location downloader database configuration APIs added in 8.16 to support more types" + +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_3" + ignore: 404 + +--- +"Test adding, getting, and removing ip location databases": + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-Country", + "maxmind": { + "account_id": "4321" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_2" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + catch: /illegal_argument_exception/ + ingest.put_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + body: > + { + "name": "GeoIP2-City", + "web": { + } + } + + - do: + ingest.put_ip_location_database: + id: "my_database_3" + body: > + { + "name": "standard_privacy", + "ipinfo": { + } + } + - match: { acknowledged: true } + + - do: + ingest.get_ip_location_database: + id: "my_database_1" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_1" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-Country" } + - match: { databases.0.database.maxmind.account_id: "4321" } + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 7 } + + - do: + ingest.get_ip_location_database: + id: "my_database_1,my_database_2" + - length: { databases: 2 } + + - do: + ingest.get_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - length: { databases: 1 } + - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" } + - gte: { databases.0.modified_date_millis: -1 } + - match: { databases.0.database.name: "MyCustomGeoLite2-City" } + + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + + - do: + catch: /resource_not_found_exception/ + ingest.delete_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 6 } + + - do: + ingest.get_ip_location_database: + id: "my_database_2" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_2" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-City" } + - match: { databases.0.database.maxmind.account_id: "1234" } + + - do: + ingest.get_ip_location_database: + id: "my_database_3" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_3" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "standard_privacy" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json new file mode 100644 index 0000000000000..e97d1da276906 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json @@ -0,0 +1,31 @@ +{ + "ingest.delete_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-ip-location-database-api.html", + "description":"Deletes an ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to delete" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json new file mode 100644 index 0000000000000..a2e42fe6c8e59 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json @@ -0,0 +1,37 @@ +{ + "ingest.get_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-ip-location-database-api.html", + "description":"Returns the specified ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database", + "methods":[ + "GET" + ] + }, + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to get; use `*` to get all ip location database configurations" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json new file mode 100644 index 0000000000000..18487969b1a90 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json @@ -0,0 +1,36 @@ +{ + "ingest.put_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-ip-location-database-api.html", + "description":"Puts the configuration for a ip location database to be downloaded" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "PUT" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The id of the database configuration" + } + } + } + ] + }, + "params":{ + }, + "body":{ + "description":"The database configuration definition", + "required":true + } + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java index 1933d285d7870..77b11357d79b1 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java @@ -22,7 +22,13 @@ public class IngestGeoIpFeatures implements FeatureSpecification { "get_database_configuration_action.multi_node" ); + public static final NodeFeature PUT_DATABASE_CONFIGURATION_ACTION_IPINFO = new NodeFeature("put_database_configuration_action.ipinfo"); + public Set getFeatures() { - return Set.of(GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE); + return Set.of( + GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, + GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE, + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO + ); } }