From 2c9451933f823852c1fb0d3fe2dbf84f054db324 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Thu, 27 Jun 2024 13:40:16 +0200 Subject: [PATCH] [processor/geoip] Add maxmind geoprovider (#33451) **Description:** This PR adds an initial implementation for the MaxMind GeoIP provider for later usage in the `geoipprocessor`. The processor is still a nop, as no provider can be configured yet. Providers configuration will be added in https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/33268. - Internal package for temporal attributes conventions reference (should be removed in favor of https://github.com/open-telemetry/semantic-conventions/pull/1116): `processor/geoipprocessor/internal/convention/attributes.go` - [geoip2-golang](https://github.com/oschwald/geoip2-golang) package used as a client for data retrieval. See recommended MaxMind clients: https://dev.maxmind.com/geoip/docs/databases#api-clients **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32663 **Testing:** Database files are generated before running the unit tests [using MaxMind writer.](https://github.com/maxmind/MaxMind-DB/tree/0a9c1aa26cd7b91bee2efe27e7d174797d8211a6/test-data#how-to-generate-test-data). The generation time of those files seems not to be an issue (0.189s): ``` $ go test -v === RUN TestInvalidNewProvider --- PASS: TestInvalidNewProvider (0.00s) === RUN TestProviderLocation === PAUSE TestProviderLocation === CONT TestProviderLocation === RUN TestProviderLocation/nil_IP_address === RUN TestProviderLocation/unsupported_database_type === RUN TestProviderLocation/no_IP_metadata_in_database === RUN TestProviderLocation/all_attributes_should_be_present_for_IPv4_using_GeoLite2-City_database === RUN TestProviderLocation/subset_attributes_for_IPv6_IP_using_GeoIP2-City_database --- PASS: TestProviderLocation (0.00s) --- PASS: TestProviderLocation/nil_IP_address (0.00s) --- PASS: TestProviderLocation/unsupported_database_type (0.00s) --- PASS: TestProviderLocation/no_IP_metadata_in_database (0.00s) --- PASS: TestProviderLocation/all_attributes_should_be_present_for_IPv4_using_GeoLite2-City_database (0.00s) --- PASS: TestProviderLocation/subset_attributes_for_IPv6_IP_using_GeoIP2-City_database (0.00s) PASS ok github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider/maxmindprovider 0.189s ``` Testing database files are removed on sucessful test execution. **Documentation:** README.md file --- .chloggen/add_maxmind_geoprovider.yaml | 27 ++++ processor/geoipprocessor/geoip_processor.go | 22 ++- .../geoipprocessor/geoip_processor_test.go | 17 +++ processor/geoipprocessor/go.mod | 5 + processor/geoipprocessor/go.sum | 10 ++ .../internal/convention/attributes.go | 40 ++++++ .../internal/provider/geoipprovider.go | 6 + .../provider/maxmindprovider/README.md | 14 ++ .../provider/maxmindprovider/config.go | 27 ++++ .../provider/maxmindprovider/provider.go | 103 ++++++++++++++ .../provider/maxmindprovider/provider_test.go | 129 ++++++++++++++++++ .../testdata/GeoIP2-Anonymous-IP-Test.json | 1 + .../testdata/GeoIP2-City-Test.json | 54 ++++++++ .../testdata/GeoIP2-Connection-Type-Test.json | 1 + .../testdata/GeoIP2-Country-Test.json | 1 + .../testdata/GeoIP2-DensityIncome-Test.json | 1 + .../testdata/GeoIP2-Domain-Test.json | 1 + .../testdata/GeoIP2-Enterprise-Test.json | 1 + .../testdata/GeoIP2-ISP-Test.json | 1 + .../GeoIP2-Precision-Enterprise-Test.json | 1 + .../testdata/GeoIP2-Static-IP-Score-Test.json | 1 + .../testdata/GeoIP2-User-Count-Test.json | 1 + .../testdata/GeoLite2-ASN-Test.json | 1 + .../testdata/GeoLite2-City-Test.json | 83 +++++++++++ .../testdata/GeoLite2-Country-Test.json | 1 + 25 files changed, 546 insertions(+), 3 deletions(-) create mode 100644 .chloggen/add_maxmind_geoprovider.yaml create mode 100644 processor/geoipprocessor/internal/convention/attributes.go create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/README.md create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/config.go create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/provider.go create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/provider_test.go create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Anonymous-IP-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-City-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Connection-Type-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Country-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-DensityIncome-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Domain-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Enterprise-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-ISP-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Precision-Enterprise-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Static-IP-Score-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-User-Count-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-ASN-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-City-Test.json create mode 100644 processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-Country-Test.json diff --git a/.chloggen/add_maxmind_geoprovider.yaml b/.chloggen/add_maxmind_geoprovider.yaml new file mode 100644 index 000000000000..050e2e2fb479 --- /dev/null +++ b/.chloggen/add_maxmind_geoprovider.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: geoipprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add MaxMind geoip provider for GeoIP2-City and GeoLite2-City databases. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32663] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/processor/geoipprocessor/geoip_processor.go b/processor/geoipprocessor/geoip_processor.go index 098ca0e515a9..def4e0b7bc00 100644 --- a/processor/geoipprocessor/geoip_processor.go +++ b/processor/geoipprocessor/geoip_processor.go @@ -6,6 +6,7 @@ package geoipprocessor // import "github.com/open-telemetry/opentelemetry-collec import ( "context" "errors" + "fmt" "net" "go.opentelemetry.io/collector/pdata/pcommon" @@ -17,7 +18,11 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider" ) -var errIPNotFound = errors.New("no IP address found in the resource attributes") +var ( + errIPNotFound = errors.New("no IP address found in the resource attributes") + errParseIP = errors.New("could not parse IP address") + errUnspecifiedIP = errors.New("unspecified address") +) // newGeoIPProcessor creates a new instance of geoIPProcessor with the specified fields. type geoIPProcessor struct { @@ -31,15 +36,26 @@ func newGeoIPProcessor(resourceAttributes []attribute.Key) *geoIPProcessor { } } +// parseIP parses a string to a net.IP type and returns an error if the IP is invalid or unspecified. +func parseIP(strIP string) (net.IP, error) { + ip := net.ParseIP(strIP) + if ip == nil { + return nil, fmt.Errorf("%w address: %s", errParseIP, strIP) + } else if ip.IsUnspecified() { + return nil, fmt.Errorf("%w address: %s", errUnspecifiedIP, strIP) + } + return ip, nil +} + // ipFromResourceAttributes extracts an IP address from the given resource's attributes based on the specified fields. // It returns the first IP address if found, or an error if no valid IP address is found. func ipFromResourceAttributes(attributes []attribute.Key, resource pcommon.Resource) (net.IP, error) { for _, attr := range attributes { if ipField, found := resource.Attributes().Get(string(attr)); found { - ipAttribute := net.ParseIP(ipField.AsString()) // The attribute might contain a domain name. Skip any net.ParseIP error until we have a fine-grained error propagation strategy. // TODO: propagate an error once error_mode configuration option is available (e.g. transformprocessor) - if ipAttribute != nil { + ipAttribute, err := parseIP(ipField.AsString()) + if err == nil && ipAttribute != nil { return ipAttribute, nil } } diff --git a/processor/geoipprocessor/geoip_processor_test.go b/processor/geoipprocessor/geoip_processor_test.go index 5418228d851c..9404ef832873 100644 --- a/processor/geoipprocessor/geoip_processor_test.go +++ b/processor/geoipprocessor/geoip_processor_test.go @@ -131,6 +131,23 @@ func TestProcessPdata(t *testing.T) { }), }, }, + { + name: "default source.ip attribute with an unspecified IP address should be skipped", + resourceAttributes: defaultResourceAttributes, + initResourceAttributes: []generateResourceFunc{ + withAttributes([]attribute.KeyValue{ + attribute.String(string(semconv.SourceAddressKey), "0.0.0.0"), + }), + }, + geoLocationMock: func(context.Context, net.IP) (attribute.Set, error) { + return attribute.NewSet([]attribute.KeyValue{attribute.String("geo.city_name", "barcelona")}...), nil + }, + expectedResourceAttributes: []generateResourceFunc{ + withAttributes([]attribute.KeyValue{ + attribute.String(string(semconv.SourceAddressKey), "0.0.0.0"), + }), + }, + }, { name: "custom resource attribute", resourceAttributes: []attribute.Key{"ip"}, diff --git a/processor/geoipprocessor/go.mod b/processor/geoipprocessor/go.mod index 916abeafcb52..c1e93b7ebb3a 100644 --- a/processor/geoipprocessor/go.mod +++ b/processor/geoipprocessor/go.mod @@ -28,11 +28,15 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/maxmind/MaxMind-DB v0.0.0-20240605211347-880f6b4b5eb6 + github.com/maxmind/mmdbwriter v1.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.103.0 // indirect + github.com/oschwald/geoip2-golang v1.11.0 + github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -49,6 +53,7 @@ require ( go.opentelemetry.io/otel/trace v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + go4.org/netipx v0.0.0-20230824141953-6213f710f925 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/processor/geoipprocessor/go.sum b/processor/geoipprocessor/go.sum index e3633609863e..78979288cb9d 100644 --- a/processor/geoipprocessor/go.sum +++ b/processor/geoipprocessor/go.sum @@ -35,6 +35,10 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maxmind/MaxMind-DB v0.0.0-20240605211347-880f6b4b5eb6 h1:q3FNO1NzvowWXqFEam6FwbsgGeTXBL5FOxjMT45x2Ls= +github.com/maxmind/MaxMind-DB v0.0.0-20240605211347-880f6b4b5eb6/go.mod h1:8uyvr1CxDqu0x6NrZZ7CAs6nR0G/wSY8K+Q7uQlNwK4= +github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw= +github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -44,6 +48,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= @@ -98,6 +106,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ= +go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/processor/geoipprocessor/internal/convention/attributes.go b/processor/geoipprocessor/internal/convention/attributes.go new file mode 100644 index 000000000000..b2ae1be866e5 --- /dev/null +++ b/processor/geoipprocessor/internal/convention/attributes.go @@ -0,0 +1,40 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package conventions // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention" + +// TODO: replace for semconv once https://github.com/open-telemetry/semantic-conventions/issues/1033 is closed. +const ( + // AttributeGeoCityName represents the attribute name for the city name in geographical data. + AttributeGeoCityName = "geo.city_name" + + // AttributeGeoPostalCode represents the attribute name for the city postal code. + AttributeGeoPostalCode = "geo.postal_code" + + // AttributeGeoCountryName represents the attribute name for the country name in geographical data. + AttributeGeoCountryName = "geo.country_name" + + // AttributeGeoCountryIsoCode represents the attribute name for the Two-letter ISO Country Code. + AttributeGeoCountryIsoCode = "geo.country_iso_code" + + // AttributeGeoContinentName represents the attribute name for the continent name in geographical data. + AttributeGeoContinentName = "geo.continent_name" + + // AttributeGeoContinentIsoCode represents the attribute name for the Two-letter Continent Code. + AttributeGeoContinentCode = "geo.continent_code" + + // AttributeGeoRegionName represents the attribute name for the region name in geographical data. + AttributeGeoRegionName = "geo.region_name" + + // AttributeGeoRegionIsoCode represents the attribute name for the Two-letter ISO Region Code. + AttributeGeoRegionIsoCode = "geo.region_iso_code" + + // AttributeGeoTimezone represents the attribute name for the timezone. + AttributeGeoTimezone = "geo.timezone" + + // AttributeGeoLocationLat represents the attribute name for the latitude. + AttributeGeoLocationLat = "geo.location.lat" + + // AttributeGeoLocationLon represents the attribute name for the longitude. + AttributeGeoLocationLon = "geo.location.lon" +) diff --git a/processor/geoipprocessor/internal/provider/geoipprovider.go b/processor/geoipprocessor/internal/provider/geoipprovider.go index e75c4d447922..db52a7573a0b 100644 --- a/processor/geoipprocessor/internal/provider/geoipprovider.go +++ b/processor/geoipprocessor/internal/provider/geoipprovider.go @@ -7,9 +7,15 @@ import ( "context" "net" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/otel/attribute" ) +// Config is the configuration of a GeoIPProvider. +type Config interface { + component.ConfigValidator +} + // GeoIPProvider defines methods for obtaining the geographical location based on the provided IP address. type GeoIPProvider interface { // Location returns a set of attributes representing the geographical location for the given IP address. It requires a context for managing request lifetime. diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/README.md b/processor/geoipprocessor/internal/provider/maxmindprovider/README.md new file mode 100644 index 000000000000..70857fe0cb81 --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/README.md @@ -0,0 +1,14 @@ +# MaxMind GeoIP Provider + +This package provides a MaxMind GeoIP provider for use with the OpenTelemetry GeoIP processor. It leverages the [geoip2-golang package](https://github.com/oschwald/geoip2-golang) to query geographical information associated with IP addresses from MaxMind databases. See recommended clients: https://dev.maxmind.com/geoip/docs/databases#api-clients + +# Features + +- Supports GeoIP2-City and GeoLite2-City database types. +- Retrieves and returns geographical metadata for a given IP address. The generated attributes follow the internal [Geo conventions](../../convention/attributes.go). + +## Configuration + +The following configuration must be provided: + +- `database_path`: local file path to a GeoIP2-City or GeoLite2-City database. diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/config.go b/processor/geoipprocessor/internal/provider/maxmindprovider/config.go new file mode 100644 index 000000000000..88c6666a5c64 --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/config.go @@ -0,0 +1,27 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package maxmind // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider/maxmindprovider" + +import ( + "errors" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider" +) + +// Config defines configuration for MaxMind provider. +type Config struct { + // DatabasePath section allows specifying a local GeoIP database + // file to retrieve the geographical metadata from. + DatabasePath string `mapstructure:"database_path"` +} + +var _ provider.Config = (*Config)(nil) + +// Validate implements provider.Config. +func (c *Config) Validate() error { + if c.DatabasePath == "" { + return errors.New("a local geoIP database path must be provided") + } + return nil +} diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/provider.go b/processor/geoipprocessor/internal/provider/maxmindprovider/provider.go new file mode 100644 index 000000000000..2a301549f38d --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/provider.go @@ -0,0 +1,103 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package maxmind // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider/maxmindprovider" + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/oschwald/geoip2-golang" + "go.opentelemetry.io/otel/attribute" + + conventions "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider" +) + +var ( + // defaultLanguageCode specifies English as the default Geolocation language code, see https://dev.maxmind.com/geoip/docs/web-services/responses#languages + defaultLanguageCode = "en" + geoIP2CityDBType = "GeoIP2-City" + geoLite2CityDBType = "GeoLite2-City" + + errUnsupportedDB = errors.New("unsupported geo IP database type") + errNoMetadataFound = errors.New("no geo IP metadata found") +) + +type maxMindProvider struct { + geoReader *geoip2.Reader + // language code to be used in name retrieval, e.g. "en" or "pt-BR" + langCode string +} + +var _ provider.GeoIPProvider = (*maxMindProvider)(nil) + +func newMaxMindProvider(cfg *Config) (*maxMindProvider, error) { + geoReader, err := geoip2.Open(cfg.DatabasePath) + if err != nil { + return nil, fmt.Errorf("could not open geoip database: %w", err) + } + + return &maxMindProvider{geoReader: geoReader, langCode: defaultLanguageCode}, nil +} + +// Location implements provider.GeoIPProvider for MaxMind. If a non City database type is used or no metadata is found in the database, an error will be returned. +func (g *maxMindProvider) Location(_ context.Context, ipAddress net.IP) (attribute.Set, error) { + switch g.geoReader.Metadata().DatabaseType { + case geoIP2CityDBType, geoLite2CityDBType: + attrs, err := g.cityAttributes(ipAddress) + if err != nil { + return attribute.Set{}, err + } else if len(*attrs) == 0 { + return attribute.Set{}, errNoMetadataFound + } + return attribute.NewSet(*attrs...), nil + default: + return attribute.Set{}, fmt.Errorf("%w type: %s", errUnsupportedDB, g.geoReader.Metadata().DatabaseType) + } +} + +// cityAttributes returns a list of key-values containing geographical metadata associated to the provided IP. The key names are populated using the internal geo IP conventions package. If the an invalid or nil IP is provided, an error is returned. +func (g *maxMindProvider) cityAttributes(ipAddress net.IP) (*[]attribute.KeyValue, error) { + attributes := make([]attribute.KeyValue, 0, 11) + + city, err := g.geoReader.City(ipAddress) + if err != nil { + return nil, err + } + + // The exact set of top-level keys varies based on the particular GeoIP2 web service you are using. If a key maps to an undefined or empty value, it is not included in the JSON object. The following anonymous function appends the given key-value only if the value is not empty. + appendIfNotEmpty := func(keyName, value string) { + if value != "" { + attributes = append(attributes, attribute.String(keyName, value)) + } + } + + // city + appendIfNotEmpty(conventions.AttributeGeoCityName, city.City.Names[g.langCode]) + // country + appendIfNotEmpty(conventions.AttributeGeoCountryName, city.Country.Names[g.langCode]) + appendIfNotEmpty(conventions.AttributeGeoCountryIsoCode, city.Country.IsoCode) + // continent + appendIfNotEmpty(conventions.AttributeGeoContinentName, city.Continent.Names[g.langCode]) + appendIfNotEmpty(conventions.AttributeGeoContinentCode, city.Continent.Code) + // postal code + appendIfNotEmpty(conventions.AttributeGeoPostalCode, city.Postal.Code) + // region + if len(city.Subdivisions) > 0 { + // The most specific subdivision is located at the last array position, see https://github.com/maxmind/GeoIP2-java/blob/2fe4c65424fed2c3c2449e5530381b6452b0560f/src/main/java/com/maxmind/geoip2/model/AbstractCityResponse.java#L112 + mostSpecificSubdivision := city.Subdivisions[len(city.Subdivisions)-1] + appendIfNotEmpty(conventions.AttributeGeoRegionName, mostSpecificSubdivision.Names[g.langCode]) + appendIfNotEmpty(conventions.AttributeGeoRegionIsoCode, mostSpecificSubdivision.IsoCode) + } + + // location + appendIfNotEmpty(conventions.AttributeGeoTimezone, city.Location.TimeZone) + if city.Location.Latitude != 0 && city.Location.Longitude != 0 { + attributes = append(attributes, attribute.Float64(conventions.AttributeGeoLocationLat, city.Location.Latitude), attribute.Float64(conventions.AttributeGeoLocationLon, city.Location.Longitude)) + } + + return &attributes, err +} diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/provider_test.go b/processor/geoipprocessor/internal/provider/maxmindprovider/provider_test.go new file mode 100644 index 000000000000..c9de227a8f6f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/provider_test.go @@ -0,0 +1,129 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package maxmind + +import ( + "context" + "net" + "os" + "testing" + + "github.com/maxmind/MaxMind-DB/pkg/writer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + + conventions "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention" +) + +func TestInvalidNewProvider(t *testing.T) { + _, err := newMaxMindProvider(&Config{}) + require.ErrorContains(t, err, "could not open geoip database: open : no such file or directory") + + _, err = newMaxMindProvider(&Config{DatabasePath: "no valid path"}) + require.ErrorContains(t, err, "could not open geoip database: open no valid path: no such file or directory") +} + +// generateLocalDB generates *.mmdb databases files given a source directory data. It uses a the writer functionality provided by MaxMind-Db/pkg/writer +func generateLocalDB(t *testing.T, sourceData string) string { + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + t.Fatal(err) + } + + w, err := writer.New(sourceData, tmpDir) + if err != nil { + t.Fatal(err) + } + + err = w.WriteGeoIP2TestDB() + if err != nil { + t.Fatal(err) + } + + return tmpDir +} + +// TestProviderLocation asserts that the MaxMind provider adds the geo location data given an IP. +func TestProviderLocation(t *testing.T) { + tmpDBfiles := generateLocalDB(t, "./testdata") + defer os.RemoveAll(tmpDBfiles) + + t.Parallel() + + tests := []struct { + name string + testDatabase string + sourceIP net.IP + expectedAttributes attribute.Set + expectedErrMsg string + }{ + { + name: "nil IP address", + testDatabase: "GeoIP2-City-Test.mmdb", + expectedErrMsg: "IP passed to Lookup cannot be nil", + }, + { + name: "unsupported database type", + sourceIP: net.IPv4(0, 0, 0, 0), + testDatabase: "GeoIP2-ISP-Test.mmdb", + expectedErrMsg: "unsupported geo IP database type type: GeoIP2-ISP", + }, + { + name: "no IP metadata in database", + sourceIP: net.IPv4(0, 0, 0, 0), + testDatabase: "GeoIP2-City-Test.mmdb", + expectedErrMsg: "no geo IP metadata found", + }, + { + name: "all attributes should be present for IPv4 using GeoLite2-City database", + sourceIP: net.IPv4(1, 2, 3, 4), + testDatabase: "GeoLite2-City-Test.mmdb", + expectedAttributes: attribute.NewSet([]attribute.KeyValue{ + attribute.String(conventions.AttributeGeoCityName, "Boxford"), + attribute.String(conventions.AttributeGeoContinentCode, "EU"), + attribute.String(conventions.AttributeGeoContinentName, "Europe"), + attribute.String(conventions.AttributeGeoCountryIsoCode, "GB"), + attribute.String(conventions.AttributeGeoCountryName, "United Kingdom"), + attribute.String(conventions.AttributeGeoTimezone, "Europe/London"), + attribute.String(conventions.AttributeGeoRegionIsoCode, "WBK"), + attribute.String(conventions.AttributeGeoRegionName, "West Berkshire"), + attribute.String(conventions.AttributeGeoPostalCode, "OX1"), + attribute.Float64(conventions.AttributeGeoLocationLat, 1234), + attribute.Float64(conventions.AttributeGeoLocationLon, 5678), + }...), + }, + { + name: "subset attributes for IPv6 IP using GeoIP2-City database", + sourceIP: net.ParseIP("2001:220::"), + testDatabase: "GeoIP2-City-Test.mmdb", + expectedAttributes: attribute.NewSet([]attribute.KeyValue{ + attribute.String(conventions.AttributeGeoContinentCode, "AS"), + attribute.String(conventions.AttributeGeoContinentName, "Asia"), + attribute.String(conventions.AttributeGeoCountryIsoCode, "KR"), + attribute.String(conventions.AttributeGeoCountryName, "South Korea"), + attribute.String(conventions.AttributeGeoTimezone, "Asia/Seoul"), + attribute.Float64(conventions.AttributeGeoLocationLat, 1), + attribute.Float64(conventions.AttributeGeoLocationLon, 1), + }...), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // prepare provider + provider, err := newMaxMindProvider(&Config{DatabasePath: tmpDBfiles + "/" + tt.testDatabase}) + assert.NoError(t, err) + + // assert metrics + actualAttributes, err := provider.Location(context.Background(), tt.sourceIP) + if tt.expectedErrMsg != "" { + assert.EqualError(t, err, tt.expectedErrMsg) + return + } + + assert.True(t, tt.expectedAttributes.Equals(&actualAttributes)) + }) + } +} diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Anonymous-IP-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Anonymous-IP-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Anonymous-IP-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-City-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-City-Test.json new file mode 100644 index 000000000000..79ae7a667dff --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-City-Test.json @@ -0,0 +1,54 @@ +[ + { + "2001:220::/32" : { + "continent" : { + "code" : "AS", + "geoname_id" : 6255147, + "names" : { + "de" : "Asien", + "en" : "Asia", + "es" : "Asia", + "fr" : "Asie", + "ja" : "アジア", + "pt-BR" : "Ásia", + "ru" : "Азия", + "zh-CN" : "亚洲" + } + }, + "country" : { + "geoname_id" : 1835841, + "iso_code" : "KR", + "names" : { + "de" : "Republik Korea", + "en" : "South Korea", + "es" : "Corea, República de", + "fr" : "Corée du Sud", + "ja" : "大韓民国", + "pt-BR" : "Coréia, República da", + "ru" : "Южная Корея", + "zh-CN" : "韩国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 1, + "longitude" : 1, + "time_zone" : "Asia/Seoul" + }, + "registered_country" : { + "geoname_id" : 1835841, + "iso_code" : "KR", + "names" : { + "de" : "Republik Korea", + "en" : "South Korea", + "es" : "Corea, República de", + "fr" : "Corée du Sud", + "ja" : "大韓民国", + "pt-BR" : "Coréia, República da", + "ru" : "Южная Корея", + "zh-CN" : "韩国" + } + } + } + } +] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Connection-Type-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Connection-Type-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Connection-Type-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Country-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Country-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Country-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-DensityIncome-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-DensityIncome-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-DensityIncome-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Domain-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Domain-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Domain-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Enterprise-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Enterprise-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Enterprise-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-ISP-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-ISP-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-ISP-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Precision-Enterprise-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Precision-Enterprise-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Precision-Enterprise-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Static-IP-Score-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Static-IP-Score-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Static-IP-Score-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-User-Count-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-User-Count-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-User-Count-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-ASN-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-ASN-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-ASN-Test.json @@ -0,0 +1 @@ +[{}] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-City-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-City-Test.json new file mode 100644 index 000000000000..eaff5474a903 --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-City-Test.json @@ -0,0 +1,83 @@ +[ + { + "1.2.3.4/32" : { + "city" : { + "names" : { + "en" : "Boxford" + } + }, + "continent" : { + "code" : "EU", + "names" : { + "de" : "Europa", + "en" : "Europe", + "es" : "Europa", + "fr" : "Europe", + "ja" : "ヨーロッパ", + "pt-BR" : "Europa", + "ru" : "Европа", + "zh-CN" : "欧洲" + } + }, + "country" : { + "geoname_id" : 2635167, + "iso_code" : "GB", + "names" : { + "de" : "Vereinigtes Königreich", + "en" : "United Kingdom", + "es" : "Reino Unido", + "fr" : "Royaume-Uni", + "ja" : "イギリス", + "pt-BR" : "Reino Unido", + "ru" : "Великобритания", + "zh-CN" : "英国" + } + }, + "location" : { + "accuracy_radius" : 100, + "latitude" : 1234, + "longitude" : 5678, + "time_zone" : "Europe/London" + }, + "postal" : { + "code" : "OX1" + }, + "registered_country" : { + "geoname_id" : 3017382, + "is_in_european_union" : true, + "iso_code" : "FR", + "names" : { + "de" : "Frankreich", + "en" : "France", + "es" : "Francia", + "fr" : "France", + "ja" : "フランス共和国", + "pt-BR" : "França", + "ru" : "Франция", + "zh-CN" : "法国" + } + }, + "subdivisions" : [ + { + "geoname_id" : 6269131, + "iso_code" : "ENG", + "names" : { + "en" : "England", + "es" : "Inglaterra", + "fr" : "Angleterre", + "pt-BR" : "Inglaterra" + } + }, + { + "geoname_id" : 3333217, + "iso_code" : "WBK", + "names" : { + "en" : "West Berkshire", + "ru" : "Западный Беркшир", + "zh-CN" : "西伯克郡" + } + } + ] + } + } +] diff --git a/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-Country-Test.json b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-Country-Test.json new file mode 100644 index 000000000000..93d51406d63f --- /dev/null +++ b/processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoLite2-Country-Test.json @@ -0,0 +1 @@ +[{}]