diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java new file mode 100644 index 0000000000000..5710a20277527 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.ESTestCase; + +import java.util.Set; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class DatabaseTests extends ESTestCase { + + public void testDatabasePropertyInvariants() { + // the city database is like a specialization of the country database + assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); + + // the isp database is like a specialization of the asn database + assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); + assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); + + // the enterprise database is like these other databases joined together + for (Database type : Set.of( + Database.City, + Database.Country, + Database.Asn, + Database.AnonymousIp, + Database.ConnectionType, + Database.Domain, + Database.Isp + )) { + assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); + } + // but in terms of the default fields, it's like a drop-in replacement for the city database + // n.b. this is just a choice we decided to make here at Elastic + assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 50b59c26749fc..e96bdbd6314b2 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.ingest.geoip; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; @@ -24,14 +23,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; @@ -57,71 +54,6 @@ public void cleanup() throws IOException { IOUtils.rm(tmpDir); } - public void testDatabasePropertyInvariants() { - // the city database is like a specialization of the country database - assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); - assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); - - // the isp database is like a specialization of the asn database - assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); - assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); - - // the enterprise database is like these other databases joined together - for (Database type : Set.of( - Database.City, - Database.Country, - Database.Asn, - Database.AnonymousIp, - Database.ConnectionType, - Database.Domain, - Database.Isp - )) { - assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); - } - // but in terms of the default fields, it's like a drop-in replacement for the city database - // n.b. this is just a choice we decided to make here at Elastic - assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); - } - - public void testCity() throws Exception { - String ip = "8.8.8.8"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(12)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("timezone"), equalTo("America/Chicago")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - public void testNullValueWithIgnoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, @@ -208,369 +140,6 @@ public void testNonExistentWithoutIgnoreMissing() { assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]")); } - public void testCity_withIpV6() throws Exception { - String ip = "2602:306:33d3:8000::3257:9652"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(16)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-FL")); - assertThat(geoData.get("region_name"), equalTo("Florida")); - assertThat(geoData.get("city_name"), equalTo("Homestead")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d))); - assertThat(geoData.get("accuracy_radius"), equalTo(50)); - assertThat(geoData.get("postal_code"), equalTo("33035")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - - public void testCityWithMissingLocation() throws Exception { - String ip = "80.231.5.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo(ip)); - } - - public void testCountry() throws Exception { - String ip = "82.170.213.79"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Country), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(9)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(true)); - assertThat(geoData.get("country_iso_code"), equalTo("NL")); - assertThat(geoData.get("country_name"), equalTo("Netherlands")); - assertThat(geoData.get("continent_code"), equalTo("EU")); - assertThat(geoData.get("continent_name"), equalTo("Europe")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(true)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("NL")); - assertThat(geoData.get("registered_country_name"), equalTo("Netherlands")); - } - - public void testCountryWithMissingLocation() throws Exception { - String ip = "80.231.5.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Country), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo(ip)); - } - - public void testAsn() throws Exception { - String ip = "82.171.64.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-ASN.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Asn), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(4)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(1136L)); - assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); - assertThat(geoData.get("network"), equalTo("82.168.0.0/14")); - } - - public void testAnonymmousIp() throws Exception { - String ip = "81.2.69.1"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Anonymous-IP-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.AnonymousIp), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(7)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("hosting_provider"), equalTo(true)); - assertThat(geoData.get("tor_exit_node"), equalTo(true)); - assertThat(geoData.get("anonymous_vpn"), equalTo(true)); - assertThat(geoData.get("anonymous"), equalTo(true)); - assertThat(geoData.get("public_proxy"), equalTo(true)); - assertThat(geoData.get("residential_proxy"), equalTo(true)); - } - - public void testConnectionType() throws Exception { - String ip = "214.78.120.5"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Connection-Type-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.ConnectionType), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("connection_type"), equalTo("Satellite")); - } - - public void testDomain() throws Exception { - String ip = "69.219.64.2"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Domain-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Domain), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("domain"), equalTo("ameritech.net")); - } - - public void testEnterprise() throws Exception { - String ip = "74.209.24.4"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Enterprise-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Enterprise), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(33)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_confidence"), equalTo(99)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-NY")); - assertThat(geoData.get("region_name"), equalTo("New York")); - assertThat(geoData.get("city_confidence"), equalTo(11)); - assertThat(geoData.get("city_name"), equalTo("Chatham")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549))); - assertThat(geoData.get("accuracy_radius"), equalTo(27)); - assertThat(geoData.get("postal_code"), equalTo("12037")); - assertThat(geoData.get("city_confidence"), equalTo(11)); - assertThat(geoData.get("asn"), equalTo(14671L)); - assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); - assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); - assertThat(geoData.get("hosting_provider"), equalTo(false)); - assertThat(geoData.get("tor_exit_node"), equalTo(false)); - assertThat(geoData.get("anonymous_vpn"), equalTo(false)); - assertThat(geoData.get("anonymous"), equalTo(false)); - assertThat(geoData.get("public_proxy"), equalTo(false)); - assertThat(geoData.get("residential_proxy"), equalTo(false)); - assertThat(geoData.get("domain"), equalTo("frpt.net")); - assertThat(geoData.get("isp"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("user_type"), equalTo("residential")); - assertThat(geoData.get("connection_type"), equalTo("Cable/DSL")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - - public void testIsp() throws Exception { - String ip = "149.101.100.1"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-ISP-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Isp), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(8)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(6167L)); - assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART")); - assertThat(geoData.get("network"), equalTo("149.101.100.0/28")); - assertThat(geoData.get("isp"), equalTo("Verizon Wireless")); - assertThat(geoData.get("isp_organization_name"), equalTo("Verizon Wireless")); - assertThat(geoData.get("mobile_network_code"), equalTo("004")); - assertThat(geoData.get("mobile_country_code"), equalTo("310")); - } - public void testAddressIsNotInTheDatabase() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, @@ -594,9 +163,9 @@ public void testAddressIsNotInTheDatabase() throws Exception { } /** - * Don't silently do DNS lookups or anything trappy on bogus data + * Tests that an exception in the IpDataLookup is propagated out of the GeoIpProcessor's execute method */ - public void testInvalid() { + public void testExceptionPropagates() { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, randomAlphaOfLength(10), diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java new file mode 100644 index 0000000000000..aca6b3564abb3 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java @@ -0,0 +1,303 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class MaxmindIpDataLookupsTests extends ESTestCase { + + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } + + public void testCity() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "8.8.8.8"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("timezone", "America/Chicago"), + entry("location", Map.of("lat", 37.751d, "lon", -97.822d)), + entry("accuracy_radius", 1000), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCity_withIpV6() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "2602:306:33d3:8000::3257:9652"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-FL"), + entry("region_name", "Florida"), + entry("city_name", "Homestead"), + entry("postal_code", "33035"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 25.4573d, "lon", -80.4572d)), + entry("accuracy_radius", 50), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCityWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testCountry() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "82.170.213.79"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", true), + entry("country_iso_code", "NL"), + entry("country_name", "Netherlands"), + entry("continent_code", "EU"), + entry("continent_name", "Europe"), + entry("registered_country_in_european_union", true), + entry("registered_country_iso_code", "NL"), + entry("registered_country_name", "Netherlands") + ) + ); + } + + /** + * Don't silently do DNS lookups or anything trappy on bogus data + */ + public void testInvalid() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "www.google.com"; + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + IpDataLookup lookup = new MaxmindIpDataLookups.Country(Database.Country.properties()); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> lookup.getData(loader, ip)); + assertThat(e.getMessage(), containsString("not an IP string literal")); + } + } + + public void testCountryWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testAsn() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-ASN.mmdb"; + String ip = "82.171.64.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Asn(Database.Asn.properties()), + Map.ofEntries(entry("ip", ip), entry("organization_name", "KPN B.V."), entry("asn", 1136L), entry("network", "82.168.0.0/14")) + ); + } + + public void testAnonymousIp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Anonymous-IP-Test.mmdb"; + String ip = "81.2.69.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.AnonymousIp(Database.AnonymousIp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting_provider", true), + entry("tor_exit_node", true), + entry("anonymous_vpn", true), + entry("anonymous", true), + entry("public_proxy", true), + entry("residential_proxy", true) + ) + ); + } + + public void testConnectionType() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Connection-Type-Test.mmdb"; + String ip = "214.78.120.5"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.ConnectionType(Database.ConnectionType.properties()), + Map.ofEntries(entry("ip", ip), entry("connection_type", "Satellite")) + ); + } + + public void testDomain() { + String databaseName = "GeoIP2-Domain-Test.mmdb"; + String ip = "69.219.64.2"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Domain(Database.Domain.properties()), + Map.ofEntries(entry("ip", ip), entry("domain", "ameritech.net")) + ); + } + + public void testEnterprise() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Enterprise-Test.mmdb"; + String ip = "74.209.24.4"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Enterprise(Database.Enterprise.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_confidence", 99), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-NY"), + entry("region_name", "New York"), + entry("city_confidence", 11), + entry("city_name", "Chatham"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 42.3478, "lon", -73.5549)), + entry("accuracy_radius", 27), + entry("postal_code", "12037"), + entry("postal_confidence", 11), + entry("asn", 14671L), + entry("organization_name", "FairPoint Communications"), + entry("network", "74.209.16.0/20"), + entry("hosting_provider", false), + entry("tor_exit_node", false), + entry("anonymous_vpn", false), + entry("anonymous", false), + entry("public_proxy", false), + entry("residential_proxy", false), + entry("domain", "frpt.net"), + entry("isp", "Fairpoint Communications"), + entry("isp_organization_name", "Fairpoint Communications"), + entry("user_type", "residential"), + entry("connection_type", "Cable/DSL"), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testIsp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-ISP-Test.mmdb"; + String ip = "149.101.100.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Isp(Database.Isp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("asn", 6167L), + entry("organization_name", "CELLCO-PART"), + entry("network", "149.101.100.0/28"), + entry("isp", "Verizon Wireless"), + entry("isp_organization_name", "Verizon Wireless"), + entry("mobile_network_code", "004"), + entry("mobile_country_code", "310") + ) + ); + } + + private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map expected) { + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + Map actual = lookup.getData(loader, ip); + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new String[0])) + ); + for (Map.Entry entry : expected.entrySet()) { + assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue())); + } + } catch (AssertionError e) { + fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip); + } catch (Exception e) { + fail(e, "Exception for database [%s] with address [%s]", databaseName, ip); + } + } + + private DatabaseReaderLazyLoader loader(final String databaseName) { + Path path = tmpDir.resolve(databaseName); + copyDatabase(databaseName, path); + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null); + } +}