Update GeoIP processor documentation

client/rest-high-level/build.gradle

@@ -88,7 +88,7 @@ testClusters.all {
   testDistribution = 'DEFAULT'
   systemProperty 'es.scripting.update.ctx_in_params', 'false'
   systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
-  setting 'geoip.downloader.enabled', 'false'
+  setting 'ingest.geoip.downloader.enabled', 'false'
   setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'

docs/build.gradle

@@ -43,6 +43,8 @@ testClusters.matching { it.name == "integTest"}.configureEach {
   if (singleNode().testDistribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
+    setting 'ingest.geoip.downloader.enabled', 'false'
+    systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
     systemProperty 'es.shutdown_feature_flag_enabled', 'true'
     keystorePassword 'keystore-password'
   }

docs/reference/ingest/apis/geoip-stats-api.asciidoc

@@ -0,0 +1,94 @@
+[[geoip-stats-api]]
+=== GeoIP stats API
+++++
+<titleabbrev>GeoIP stats</titleabbrev>
+++++
+
+Gets download statistics for GeoIP2 databases used with the
+<<geoip-processor,`geoip` processor>>.
+
+[source,console]
+----
+GET _ingest/geoip/stats
+----
+
+[[geoip-stats-api-request]]
+==== {api-request-title}
+
+`GET _ingest/geoip/stats`
+
+[[geoip-stats-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `monitor` or
+`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+* If <<ingest-geoip-downloader-enabled,`ingest.geoip.downloader.enabled`>> is
+disabled, this API returns zero values and an empty `nodes` object.
+
+[role="child_attributes"]
+[[geoip-stats-api-response-body]]
+==== {api-response-body-title}
+
+`stats`::
+(object)
+Download statistics for all GeoIP2 databases.
++
+.Properties of `stats`
+[%collapsible%open]
+====
+`successful_downloads`::
+(integer)
+Total number of successful database downloads.
+
+`failed_downloads`::
+(integer)
+Total number of failed database downloads.
+
+`total_download_time`::
+(integer)
+Total milliseconds spent downloading databases.
+
+`database_count`::
+(integer)
+Current number of GeoIP2 databases available for use.
+
+`skipped_updates`::
+(integer)
+Total number of database updates skipped.
+====
+
+`nodes`::
+(object)
+Current downloaded GeoIP2 databases and related files for each node.
++
+.Properties of `nodes`
+[%collapsible%open]
+====
+`<node_id>`::
+(object)
+Current downloaded databases and related files for the node. The field key is
+the node ID.
++
+.Properties of `<node_id>`
+[%collapsible%open]
+=====
+`databases`::
+(array of objects)
+Current databases downloaded for the node.
++
+.Properties of `databases` objects
+[%collapsible%open]
+======
+`name`::
+(string)
+Name of the database.
+======
+
+`files_in_temp`::
+(array of strings)
+Current databases files, including related license files. {es} stores
+these files in the node's <<es-tmpdir,temporary directory>>:
+`$ES_TMPDIR/geoip-databases/<node_id>`.
+=====
+====

docs/reference/ingest/apis/index.asciidoc

@@ -1,15 +1,29 @@
 [[ingest-apis]]
 == Ingest APIs
 
-The following ingest APIs are available for managing pipelines:
+Use ingest APIs to manage tasks and resources related to <<ingest,ingest
+pipelines>> and processors.
 
-* <<put-pipeline-api>> to add or update a pipeline
-* <<get-pipeline-api>> to return a specific pipeline
+[[ingest-pipeline-apis]]
+=== Ingest pipeline APIs
+
+Use the following APIs to create, manage, and test ingest pipelines:
+
+* <<put-pipeline-api>> to create or update a pipeline
+* <<get-pipeline-api>> to retrieve a pipeline configuration
 * <<delete-pipeline-api>> to delete a pipeline
-* <<simulate-pipeline-api>> to simulate a call to a pipeline
+* <<simulate-pipeline-api>> to test a pipeline
+
+[[ingest-stat-apis]]
+=== Stat APIs
 
+Use the following APIs to get statistics about ingest processing:
+
+* <<geoip-stats-api>> to get download statistics for GeoIP2 databases used with
+the <<geoip-processor,`geoip` processor>>.
 
 include::put-pipeline.asciidoc[]
-include::get-pipeline.asciidoc[]
 include::delete-pipeline.asciidoc[]
+include::get-pipeline.asciidoc[]
+include::geoip-stats-api.asciidoc[]
 include::simulate-pipeline.asciidoc[]

docs/reference/ingest/processors/geoip.asciidoc

@@ -4,21 +4,20 @@
 <titleabbrev>GeoIP</titleabbrev>
 ++++
 
-The `geoip` processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases.
-This processor adds this information by default under the `geoip` field. The `geoip` processor can resolve both IPv4 and
-IPv6 addresses.
-
-The `ingest-geoip` module ships by default with the GeoLite2 City, GeoLite2 Country and GeoLite2 ASN GeoIP2 databases from Maxmind made available
-under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
-
-The `geoip` processor can run with other city, country and ASN GeoIP2 databases
-from Maxmind. The database files must be copied into the `ingest-geoip` config
-directory located at `$ES_CONFIG/ingest-geoip`. Custom database files must be
-stored uncompressed and the extension must be `-City.mmdb`, `-Country.mmdb`, or
-`-ASN.mmdb` to indicate the type of the database. These database files can not
-have the same filename as any of the built-in database names. The
-`database_file` processor option is used to specify the filename of the custom
-database to use for the processor.
+The `geoip` processor adds information about the geographical location of an
+IPv4 or IPv6 address.
+
+[[geoip-automatic-updates]]
+By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2
+ASN GeoIP2 databases from
+http://dev.maxmind.com/geoip/geoip2/geolite2/[MaxMind], shared under the
+CCA-ShareAlike 4.0 license. {es} automatically downloads and manages updates for
+these databases from the Elastic GeoIP endpoint:
+https://geoip.elastic.co/v1/database. To get download statistics for these
+updates, use the <<geoip-stats-api,GeoIP stats API>>.
+
+If your cluster can't connect to the Elastic GeoIP endpoint or you want to
+manage your own updates, see <<manage-geoip-database-updates>>.
 
 [[using-ingest-geoip]]
 ==== Using the `geoip` Processor in a Pipeline
@@ -29,7 +28,7 @@ database to use for the processor.
 |======
 | Name                   | Required  | Default                                                                            | Description
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
-| `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
+| `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the MaxMind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename referring to a database the module ships with (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb) or a custom database in the `ingest-geoip` config directory.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
@@ -300,6 +299,83 @@ GET /my_ip_locations/_search
 // TESTRESPONSE[s/"took" : 3/"took" : $body.took/]
 ////
 
+[[manage-geoip-database-updates]]
+==== Manage your own GeoIP2 database updates
+
+If you can't <<geoip-automatic-updates,automatically update>> your GeoIP2
+databases from the Elastic endpoint, you have a few options:
+
+* <<use-proxy-geoip-endpoint,Use a proxy endpoint>>
+* <<use-custom-geoip-endpoint,Use a custom endpoint>>
+* <<manually-update-geoip-databases,Manually update your GeoIP2 databases>>
+
+To disable automatic database updates entirely, use the
+<<cluster-update-settings,update cluster settings API>> to set
+<<ingest-geoip-downloader-enabled,`ingest.geoip.downloader.enabled`>> to
+`false`.
+
+[[use-proxy-geoip-endpoint]]
+**Use a proxy endpoint**
+
+If you can't connect directly to the Elastic GeoIP endpoint, consider setting up
+a secure proxy. You can then specify the proxy endpoint URL in the
+<<ingest-geoip-downloader-endpoint,`ingest.geoip.downloader.endpoint`>> setting
+of each node’s `elasticsearch.yml` file.
+
+[[use-custom-geoip-endpoint]]
+**Use a custom endpoint**
+
+You can create a service that mimics the Elastic GeoIP endpoint. You can then
+get automatic updates from this service.
+
+. Download your `.mmdb` database files from the
+http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site].
+
+. Copy your database files to a single directory.
+
+. From your {es} directory, run:
++
+[source,sh]
+----
+./bin/elasticsearch-geoip -s my/source/dir [-t target/directory]
+----
+
+. Serve the static database files from your directory. For example, you can use
+Docker to serve the files from an nginx server:
++
+[source,sh]
+----
+docker run -v my/source/dir:/usr/share/nginx/html:ro nginx
+----
+
+. Specify the service's endpoint URL in the
+<<ingest-geoip-downloader-endpoint,`ingest.geoip.downloader.endpoint`>> setting
+of each node’s `elasticsearch.yml` file.
++
+By default, {es} checks the endpoint for updates every three days. To use
+another polling interval, use the <<cluster-update-settings,update cluster
+settings API>> to set
+<<ingest-geoip-downloader-poll-interval,`ingest.geoip.downloader.poll.interval`>>.
+
+[[manually-update-geoip-databases]]
+**Manually update your GeoIP2 databases**
+
+. Use the <<cluster-update-settings,update cluster settings API>> to set
+`ingest.geoip.downloader.enabled` to `false`. This disables automatic updates
+that may overwrite your database changes.
+
+. Download your `.mmdb` database files from the
+http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site].
++
+You can also use custom city, country, and ASN `.mmdb` files. These files must
+be uncompressed and use the respective `-City.mmdb`, `-Country.mmdb`, or
+`-ASN.mmdb` extensions.
+
+. Copy the database files to `$ES_CONFIG/ingest-geoip`.
+
+. In your `geoip` processors, configure the `database_file` parameter to use a
+custom database file.
+
 [[ingest-geoip-settings]]
 ===== Node Settings
 
@@ -310,3 +386,28 @@ The `geoip` processor supports the following setting:
     The maximum number of results that should be cached. Defaults to `1000`.
 
 Note that these settings are node settings and apply to all `geoip` processors, i.e. there is one cache for all defined `geoip` processors.
+
+[[geoip-cluster-settings]]
+===== Cluster settings
+
+[[ingest-geoip-downloader-enabled]]
+`ingest.geoip.downloader.enabled`::
+(<<dynamic-cluster-setting,Dynamic>>, Boolean)
+If `true`, {es} downloads automatic updates for GeoIP2 databases from the
+`ingest.geoip.downloader.endpoint`. If `false`, {es} does not download updates
+and deletes all downloaded databases. Defaults to `true`.
+
+[[ingest-geoip-downloader-endpoint]]
+`ingest.geoip.downloader.endpoint`::
+(<<static-cluster-setting,Static>>, string)
+Endpoint URL used to download updates for GeoIP2 databases. Defaults to
+`https://geoip.elastic.co/v1/database`. {es} stores downloaded database files in
+each node's <<es-tmpdir,temporary directory>> at
+`$ES_TMPDIR/geoip-databases/<node_id>`.
+
+[[ingest-geoip-downloader-poll-interval]]
+`ingest.geoip.downloader.poll.interval`::
+(<<dynamic-cluster-setting,Dynamic>>, <<time-units,time value>>)
+How often {es} checks for GeoIP2 database updates at the
+`ingest.geoip.downloader.endpoint`. Must be greater than `1d` (one day). Defaults
+to `3d` (three days).

modules/ingest-geoip/build.gradle

@@ -59,7 +59,7 @@ tasks.named("internalClusterTest").configure {
 
 testClusters.all {
   systemProperty "es.geoip_v2_feature_flag_enabled", "true"
-  setting "geoip.downloader.enabled", "false"
+  setting "ingest.geoip.downloader.enabled", "false"
 }
 
 tasks.register("copyDefaultGeoIp2DatabaseFiles", Copy) {

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -60,9 +60,9 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
 
     public static final boolean GEOIP_V2_FEATURE_FLAG_ENABLED = "true".equals(System.getProperty("es.geoip_v2_feature_flag_enabled"));
 
-    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("geoip.downloader.poll.interval",
+    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("ingest.geoip.downloader.poll.interval",
         TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope);
-    public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("geoip.downloader.endpoint",
+    public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("ingest.geoip.downloader.endpoint",
         "https://geoip.elastic.co/v1/database", Property.NodeScope);
 
     public static final String GEOIP_DOWNLOADER = "geoip-downloader";

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -34,12 +34,12 @@
 
 /**
  * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node.
- * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'geoip.downloader.enabled' setting
+ * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'ingest.geoip.downloader.enabled' setting
  */
 public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<GeoIpTaskParams> implements ClusterStateListener {
 
-    public static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("geoip.downloader.enabled", GEOIP_V2_FEATURE_FLAG_ENABLED,
-        Setting.Property.Dynamic, Setting.Property.NodeScope);
+    public static final Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("ingest.geoip.downloader.enabled",
+        GEOIP_V2_FEATURE_FLAG_ENABLED, Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class);