[processor/geoip] Add maxmind geoprovider

processor/geoipprocessor/go.mod

@@ -14,6 +14,11 @@ require (
 	go.uber.org/goleak v1.3.0
 )
 
+require (
+	github.com/maxmind/mmdbwriter v1.0.0 // indirect
+	go4.org/netipx v0.0.0-20230824141953-6213f710f925 // indirect
+)
+
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -27,11 +32,14 @@ require (
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
 	github.com/knadh/koanf/v2 v2.1.1 // indirect
+	github.com/maxmind/MaxMind-DB v0.0.0-20240605211347-880f6b4b5eb6
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.102.0 // indirect
+	github.com/oschwald/geoip2-golang v1.11.0
+	github.com/oschwald/maxminddb-golang v1.13.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

processor/geoipprocessor/internal/convention/attributes.go

@@ -0,0 +1,40 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package conventions // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention"
+
+// TODO: replace for semconv once https://github.com/open-telemetry/semantic-conventions/issues/1033 is closed.
+const (
+	// AttributeGeoCityName represents the attribute name for the city name in geographical data.
+	AttributeGeoCityName = "geo.city_name"
+
+	// AttributeGeoPostalCode represents the attribute name for the city postal code.
+	AttributeGeoPostalCode = "geo.postal_code"
+
+	// AttributeGeoCountryName represents the attribute name for the country name in geographical data.
+	AttributeGeoCountryName = "geo.country_name"
+
+	// AttributeGeoCountryIsoCode represents the attribute name for the Two-letter ISO Country Code.
+	AttributeGeoCountryIsoCode = "geo.country_iso_code"
+
+	// AttributeGeoContinentName represents the attribute name for the continent name in geographical data.
+	AttributeGeoContinentName = "geo.continent_name"
+
+	// AttributeGeoContinentIsoCode represents the attribute name for the Two-letter Continent Code.
+	AttributeGeoContinentCode = "geo.continent_code"
+
+	// AttributeGeoRegionName represents the attribute name for the region name in geographical data.
+	AttributeGeoRegionName = "geo.region_name"
+
+	// AttributeGeoRegionIsoCode represents the attribute name for the Two-letter ISO Region Code.
+	AttributeGeoRegionIsoCode = "geo.region_iso_code"
+
+	// AttributeGeoTimezone represents the attribute name for the timezone.
+	AttributeGeoTimezone = "geo.timezone"
+
+	// AttributeGeoLocationLat represents the attribute name for the latitude.
+	AttributeGeoLocationLat = "geo.location.lat"
+
+	// AttributeGeoLocationLon represents the attribute name for the longitude.
+	AttributeGeoLocationLon = "geo.location.lon"
+)

processor/geoipprocessor/internal/provider/geoipprovider.go

@@ -7,9 +7,15 @@ import (
 	"context"
 	"net"
 
+	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/otel/attribute"
 )
 
+// Config is the configuration of a GeoIPProvider.
+type Config interface {
+	component.ConfigValidator
+}
+
 // GeoIPProvider defines methods for obtaining the geographical location based on the provided IP address.
 type GeoIPProvider interface {
 	// Location returns a set of attributes representing the geographical location for the given IP address. It requires a context for managing request lifetime.

processor/geoipprocessor/internal/provider/maxmindprovider/README.md

@@ -0,0 +1,14 @@
+# MaxMind GeoIP Provider
+
+This package provides a MaxMind GeoIP provider for use with the OpenTelemetry GeoIP processor. It leverages the [geoip2-golang package](https://github.com/oschwald/geoip2-golang) to query geographical information associated with IP addresses from MaxMind databases. See recommended clients: https://dev.maxmind.com/geoip/docs/databases#api-clients
+
+# Features
+
+- Supports GeoIP2-City and GeoLite2-City database types.
+- Retrieves and returns geographical metadata for a given IP address. The generated attributes follow the internal [Geo conventions](../../convention/attributes.go).
+
+## Configuration
+
+The following configuration must be provided:
+
+- `geoip_database_path`: local file path to a GeoIP2-City or GeoLite2-City database.

processor/geoipprocessor/internal/provider/maxmindprovider/config.go

@@ -0,0 +1,27 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package maxmind
+
+import (
+	"errors"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider"
+)
+
+// Config defines configuration for MaxMind provider.
+type Config struct {
+	// GeoIPDatabasePath section allows specifying a local GeoIP database
+	// file to retrieve the geographical metadata from.
+	GeoIPDatabasePath string `mapstructure:"geoip_database_path"`
+}
+
+var _ provider.Config = (*Config)(nil)
+
+// Validate implements provider.Config.
+func (c *Config) Validate() error {
+	if c.GeoIPDatabasePath == "" {
+		return errors.New("a local geoIP database path must be provided")
+	}
+	return nil
+}

processor/geoipprocessor/internal/provider/maxmindprovider/provider.go

@@ -0,0 +1,103 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package maxmind
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+
+	"github.com/oschwald/geoip2-golang"
+	"go.opentelemetry.io/otel/attribute"
+
+	conventions "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/provider"
+)
+
+var (
+	// defaultLocale specifies English as the default Geolocation name, see https://dev.maxmind.com/geoip/docs/web-services/responses#languages
+	defaultLocale      = "en"
+	geoIP2CityDBType   = "GeoIP2-City"
+	geoLite2CityDBType = "GeoLite2-City"
+
+	errUnsupportedDB   = errors.New("unsupported geo IP database type")
+	errNoMetadataFound = errors.New("no geo IP metadata found")
+)
+
+type maxMindProvider struct {
+	geoReader *geoip2.Reader
+	// language/locale to be used in name retrieval
+	name string
+}
+
+var _ provider.GeoIPProvider = (*maxMindProvider)(nil)
+
+func newMaxMindProvider(cfg *Config) (*maxMindProvider, error) {
+	geoReader, err := geoip2.Open(cfg.GeoIPDatabasePath)
+	if err != nil {
+		return nil, fmt.Errorf("could not open geoip database: %w", err)
+	}
+
+	return &maxMindProvider{geoReader: geoReader, name: defaultLocale}, nil
+}
+
+// Location implements provider.GeoIPProvider for MaxMind. If a non City database type is used or no metadata is found in the database, an error will be returned.
+func (g *maxMindProvider) Location(_ context.Context, ipAddress net.IP) (attribute.Set, error) {
+	switch g.geoReader.Metadata().DatabaseType {
+	case geoIP2CityDBType, geoLite2CityDBType:
+		attrs, err := g.cityAttributes(ipAddress)
+		if err != nil {
+			return attribute.Set{}, err
+		} else if len(*attrs) == 0 {
+			return attribute.Set{}, errNoMetadataFound
+		}
+		return attribute.NewSet(*attrs...), nil
+	default:
+		return attribute.Set{}, fmt.Errorf("%w type: %s", errUnsupportedDB, g.geoReader.Metadata().DatabaseType)
+	}
+}
+
+// cityAttributes returns a list of key-values containing geographical metadata associated to the provided IP. The key names are populated using the internal geo IP conventions package. If the an invalid or nil IP is provided, an error is returned.
+func (g *maxMindProvider) cityAttributes(ipAddress net.IP) (*[]attribute.KeyValue, error) {
+	attributes := make([]attribute.KeyValue, 0, 11)
+
+	city, err := g.geoReader.City(ipAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// The exact set of top-level keys varies based on the particular GeoIP2 web service you are using. If a key maps to an undefined or empty value, it is not included in the JSON object. The following anonymous function appends the given key-value only if the value is not empty.
+	appendIfNotEmpty := func(keyName, value string) {
+		if value != "" {
+			attributes = append(attributes, attribute.String(keyName, value))
+		}
+	}
+
+	// city
+	appendIfNotEmpty(conventions.AttributeGeoCityName, city.City.Names[g.name])
+	// country
+	appendIfNotEmpty(conventions.AttributeGeoCountryName, city.Country.Names[g.name])
+	appendIfNotEmpty(conventions.AttributeGeoCountryIsoCode, city.Country.IsoCode)
+	// continent
+	appendIfNotEmpty(conventions.AttributeGeoContinentName, city.Continent.Names[g.name])
+	appendIfNotEmpty(conventions.AttributeGeoContinentCode, city.Continent.Code)
+	// postal code
+	appendIfNotEmpty(conventions.AttributeGeoPostalCode, city.Postal.Code)
+	// region
+	if len(city.Subdivisions) > 0 {
+		// The most specific subdivision is located at the last array position, see https://github.com/maxmind/GeoIP2-java/blob/2fe4c65424fed2c3c2449e5530381b6452b0560f/src/main/java/com/maxmind/geoip2/model/AbstractCityResponse.java#L112
+		mostSpecificSubdivision := city.Subdivisions[len(city.Subdivisions)-1]
+		appendIfNotEmpty(conventions.AttributeGeoRegionName, mostSpecificSubdivision.Names[g.name])
+		appendIfNotEmpty(conventions.AttributeGeoRegionIsoCode, mostSpecificSubdivision.IsoCode)
+	}
+
+	// location
+	appendIfNotEmpty(conventions.AttributeGeoTimezone, city.Location.TimeZone)
+	if city.Location.Latitude != 0 && city.Location.Longitude != 0 {
+		attributes = append(attributes, attribute.Float64(conventions.AttributeGeoLocationLat, city.Location.Latitude), attribute.Float64(conventions.AttributeGeoLocationLon, city.Location.Longitude))
+	}
+
+	return &attributes, err
+}

processor/geoipprocessor/internal/provider/maxmindprovider/provider_test.go

@@ -0,0 +1,129 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package maxmind
+
+import (
+	"context"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/maxmind/MaxMind-DB/pkg/writer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+
+	conventions "github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor/internal/convention"
+)
+
+func TestInvalidNewProvider(t *testing.T) {
+	_, err := newMaxMindProvider(&Config{})
+	require.ErrorContains(t, err, "could not open geoip database: open : no such file or directory")
+
+	_, err = newMaxMindProvider(&Config{GeoIPDatabasePath: "no valid path"})
+	require.ErrorContains(t, err, "could not open geoip database: open no valid path: no such file or directory")
+}
+
+// generateLocalDB generates *.mmdb databases files given a source directory data. It uses a the writer functionality provided by MaxMind-Db/pkg/writer
+func generateLocalDB(t *testing.T, sourceData string) string {
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w, err := writer.New(sourceData, tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = w.WriteGeoIP2TestDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return tmpDir
+}
+
+// TestProviderLocation asserts that the MaxMind provider adds the geo location data given an IP.
+func TestProviderLocation(t *testing.T) {
+	tmpDBfiles := generateLocalDB(t, "./testdata")
+	defer os.RemoveAll(tmpDBfiles)
+
+	t.Parallel()
+
+	tests := []struct {
+		name               string
+		testDatabase       string
+		sourceIP           net.IP
+		expectedAttributes attribute.Set
+		expectedErrMsg     string
+	}{
+		{
+			name:           "nil IP address",
+			testDatabase:   "GeoIP2-City-Test.mmdb",
+			expectedErrMsg: "IP passed to Lookup cannot be nil",
+		},
+		{
+			name:           "unsupported database type",
+			sourceIP:       net.IPv4(0, 0, 0, 0),
+			testDatabase:   "GeoIP2-ISP-Test.mmdb",
+			expectedErrMsg: "unsupported geo IP database type type: GeoIP2-ISP",
+		},
+		{
+			name:           "no IP metadata in database",
+			sourceIP:       net.IPv4(0, 0, 0, 0),
+			testDatabase:   "GeoIP2-City-Test.mmdb",
+			expectedErrMsg: "no geo IP metadata found",
+		},
+		{
+			name:         "all attributes should be present for IPv4 using GeoLite2-City database",
+			sourceIP:     net.IPv4(1, 2, 3, 4),
+			testDatabase: "GeoLite2-City-Test.mmdb",
+			expectedAttributes: attribute.NewSet([]attribute.KeyValue{
+				attribute.String(conventions.AttributeGeoCityName, "Boxford"),
+				attribute.String(conventions.AttributeGeoContinentCode, "EU"),
+				attribute.String(conventions.AttributeGeoContinentName, "Europe"),
+				attribute.String(conventions.AttributeGeoCountryIsoCode, "GB"),
+				attribute.String(conventions.AttributeGeoCountryName, "United Kingdom"),
+				attribute.String(conventions.AttributeGeoTimezone, "Europe/London"),
+				attribute.String(conventions.AttributeGeoRegionIsoCode, "WBK"),
+				attribute.String(conventions.AttributeGeoRegionName, "West Berkshire"),
+				attribute.String(conventions.AttributeGeoPostalCode, "OX1"),
+				attribute.Float64(conventions.AttributeGeoLocationLat, 1234),
+				attribute.Float64(conventions.AttributeGeoLocationLon, 5678),
+			}...),
+		},
+		{
+			name:         "subset attributes for IPv6 IP using GeoIP2-City database",
+			sourceIP:     net.ParseIP("2001:220::"),
+			testDatabase: "GeoIP2-City-Test.mmdb",
+			expectedAttributes: attribute.NewSet([]attribute.KeyValue{
+				attribute.String(conventions.AttributeGeoContinentCode, "AS"),
+				attribute.String(conventions.AttributeGeoContinentName, "Asia"),
+				attribute.String(conventions.AttributeGeoCountryIsoCode, "KR"),
+				attribute.String(conventions.AttributeGeoCountryName, "South Korea"),
+				attribute.String(conventions.AttributeGeoTimezone, "Asia/Seoul"),
+				attribute.Float64(conventions.AttributeGeoLocationLat, 1),
+				attribute.Float64(conventions.AttributeGeoLocationLon, 1),
+			}...),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// prepare provider
+			provider, err := newMaxMindProvider(&Config{GeoIPDatabasePath: tmpDBfiles + "/" + tt.testDatabase})
+			assert.NoError(t, err)
+
+			// assert metrics
+			actualAttributes, err := provider.Location(context.Background(), tt.sourceIP)
+			if tt.expectedErrMsg != "" {
+				assert.EqualError(t, err, tt.expectedErrMsg)
+				return
+			}
+
+			assert.True(t, tt.expectedAttributes.Equals(&actualAttributes))
+		})
+	}
+}

processor/geoipprocessor/internal/provider/maxmindprovider/testdata/GeoIP2-Anonymous-IP-Test.json

@@ -0,0 +1 @@
+[{}]