Set User-Agent in http request

Signed-off-by: Heemin Kim <[email protected]>
opensearch-project · May 10, 2023 · 6b49244 · 6b49244
1 parent 91c609b
commit 6b49244
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 18 deletions.
diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java
@@ -9,6 +9,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URL;
+import java.net.URLConnection;
 import java.nio.CharBuffer;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -25,6 +26,7 @@
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.geospatial.shared.Constants;
 
 /**
  * Ip2Geo datasource manifest file object
@@ -39,7 +41,7 @@ public class DatasourceManifest {
     private static final ParseField DB_NAME_FIELD = new ParseField("db_name");
     private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash");
     private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days");
-    private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at");
+    private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at_in_epoch_milli");
     private static final ParseField PROVIDER_FIELD = new ParseField("provider");
 
     /**
@@ -114,16 +116,21 @@ public static class Builder {
         public static DatasourceManifest build(final URL url) {
             SpecialPermission.check();
             return AccessController.doPrivileged((PrivilegedAction<DatasourceManifest>) () -> {
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
-                    CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES);
-                    reader.read(charBuffer);
-                    charBuffer.flip();
-                    XContentParser parser = JsonXContent.jsonXContent.createParser(
-                        NamedXContentRegistry.EMPTY,
-                        DeprecationHandler.IGNORE_DEPRECATIONS,
-                        charBuffer.toString()
-                    );
-                    return PARSER.parse(parser, null);
+                try {
+                    URLConnection con = url.openConnection();
+                    con.addRequestProperty("User-Agent", Constants.USER_AGENT);
+                    InputStreamReader inputStreamReader = new InputStreamReader(con.getInputStream());
+                    try (BufferedReader reader = new BufferedReader(inputStreamReader)) {
+                        CharBuffer charBuffer = CharBuffer.allocate(MANIFEST_FILE_MAX_BYTES);
+                        reader.read(charBuffer);
+                        charBuffer.flip();
+                        XContentParser parser = JsonXContent.jsonXContent.createParser(
+                            NamedXContentRegistry.EMPTY,
+                            DeprecationHandler.IGNORE_DEPRECATIONS,
+                            charBuffer.toString()
+                        );
+                        return PARSER.parse(parser, null);
+                    }
                 } catch (IOException e) {
                     throw new RuntimeException(e);
                 }

diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacade.java
@@ -12,6 +12,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
+import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -52,6 +53,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.geospatial.shared.Constants;
 import org.opensearch.geospatial.shared.StashedThreadContext;
 import org.opensearch.index.query.QueryBuilders;
 
@@ -138,7 +140,9 @@ public CSVParser getDatabaseReader(final DatasourceManifest manifest) {
         return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
             try {
                 URL zipUrl = new URL(manifest.getUrl());
-                ZipInputStream zipIn = new ZipInputStream(zipUrl.openStream());
+                URLConnection con = zipUrl.openConnection();
+                con.addRequestProperty("User-Agent", Constants.USER_AGENT);
+                ZipInputStream zipIn = new ZipInputStream(con.getInputStream());
                 ZipEntry zipEntry = zipIn.getNextEntry();
                 while (zipEntry != null) {
                     if (zipEntry.getName().equalsIgnoreCase(manifest.getDbName()) == false) {

diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java
@@ -23,8 +23,7 @@ public class Ip2GeoSettings {
      */
     public static final Setting<String> DATASOURCE_ENDPOINT = Setting.simpleString(
         "plugins.geospatial.ip2geo.datasource.endpoint",
-        // TODO: This value is not correct. Update it later once CDN server is ready.
-        "https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json",
+        "https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json",
         new DatasourceEndpointValidator(),
         Setting.Property.NodeScope,
         Setting.Property.Dynamic

diff --git a/src/main/java/org/opensearch/geospatial/shared/Constants.java b/src/main/java/org/opensearch/geospatial/shared/Constants.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.geospatial.shared;
+
+import java.util.Locale;
+
+import org.opensearch.Version;
+
+public class Constants {
+    public static final String USER_AGENT = String.format(Locale.ROOT, "OpenSearch/%s vanilla", Version.CURRENT.toString());
+}
diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java
@@ -69,7 +69,7 @@ public void testPrepareRequestDefaultValue() {
         verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> {
             assertTrue(actionRequest instanceof PutDatasourceRequest);
             PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest;
-            assertEquals("https://geoip.maps.opensearch.org/v1/geolite-2/manifest.json", putDatasourceRequest.getEndpoint());
+            assertEquals("https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", putDatasourceRequest.getEndpoint());
             assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval());
             assertEquals(datasourceName, putDatasourceRequest.getName());
             isExecuted.set(true);

diff --git a/src/test/resources/ip2geo/manifest.json b/src/test/resources/ip2geo/manifest.json
@@ -3,6 +3,6 @@
   "db_name": "sample_valid.csv",
   "sha256_hash": "safasdfaskkkesadfasdf",
   "valid_for_in_days": 30,
-  "updated_at": 3134012341236,
+  "updated_at_in_epoch_milli": 3134012341236,
   "provider": "sample_provider"
 }
diff --git a/src/test/resources/ip2geo/manifest_invalid_url.json b/src/test/resources/ip2geo/manifest_invalid_url.json
@@ -3,6 +3,6 @@
   "db_name": "sample_valid.csv",
   "sha256_hash": "safasdfaskkkesadfasdf",
   "valid_for_in_days": 30,
-  "updated_at": 3134012341236,
+  "updated_at_in_epoch_milli": 3134012341236,
   "provider": "sample_provider"
 }
diff --git a/src/test/resources/ip2geo/manifest_template.json b/src/test/resources/ip2geo/manifest_template.json
@@ -3,6 +3,6 @@
   "db_name": "sample_valid.csv",
   "sha256_hash": "safasdfaskkkesadfasdf",
   "valid_for_in_days": 30,
-  "updated_at": 3134012341236,
+  "updated_at_in_epoch_milli": 3134012341236,
   "provider": "maxmind"
 }