From 2bc5bb07931e2a0f9c2a478849dc614e0983ceed Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 4 May 2023 11:14:01 -0700 Subject: [PATCH] Change field name from md5 to sha256 (#285) Signed-off-by: Heemin Kim --- .../ip2geo/common/DatasourceManifest.java | 14 ++++++------- .../ip2geo/jobscheduler/Datasource.java | 20 +++++++++---------- .../jobscheduler/DatasourceUpdateService.java | 4 ++-- .../geospatial/ip2geo/Ip2GeoTestCase.java | 2 +- .../ip2geo/common/GeoIpDataFacadeTests.java | 4 ++-- .../ip2geo/jobscheduler/DatasourceTests.java | 4 ++-- .../DatasourceUpdateServiceTests.java | 10 +++++----- src/test/resources/ip2geo/manifest.json | 2 +- .../ip2geo/manifest_invalid_url.json | 2 +- .../resources/ip2geo/manifest_template.json | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java index 0c65382f..957740a8 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/DatasourceManifest.java @@ -40,7 +40,7 @@ public class DatasourceManifest { private static final ParseField URL_FIELD = new ParseField("url"); private static final ParseField DB_NAME_FIELD = new ParseField("db_name"); - private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); private static final ParseField VALID_FOR_IN_DAYS_FIELD = new ParseField("valid_for_in_days"); private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); private static final ParseField PROVIDER_FIELD = new ParseField("provider"); @@ -56,10 +56,10 @@ public class DatasourceManifest { */ private String dbName; /** - * @param md5Hash MD5 hash value of a database file - * @return MD5 hash value of a database file + * @param sha256Hash SHA256 hash value of a database file + * @return SHA256 hash value of a database file */ - private String md5Hash; + private String sha256Hash; /** * @param validForInDays A duration in which the database file is valid to use * @return A duration in which the database file is valid to use @@ -85,17 +85,17 @@ public class DatasourceManifest { args -> { String url = (String) args[0]; String dbName = (String) args[1]; - String md5hash = (String) args[2]; + String sha256Hash = (String) args[2]; Long validForInDays = (Long) args[3]; Long updatedAt = (Long) args[4]; String provider = (String) args[5]; - return new DatasourceManifest(url, dbName, md5hash, validForInDays, updatedAt, provider); + return new DatasourceManifest(url, dbName, sha256Hash, validForInDays, updatedAt, provider); } ); static { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), URL_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DB_NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java index fc7d2e58..2f93cff5 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/Datasource.java @@ -345,7 +345,7 @@ public boolean isExpired() { */ public void setDatabase(final DatasourceManifest datasourceManifest, final List fields) { this.database.setProvider(datasourceManifest.getProvider()); - this.database.setMd5Hash(datasourceManifest.getMd5Hash()); + this.database.setSha256Hash(datasourceManifest.getSha256Hash()); this.database.setUpdatedAt(Instant.ofEpochMilli(datasourceManifest.getUpdatedAt())); this.database.setValidForInDays(datasourceManifest.getValidForInDays()); this.database.setFields(fields); @@ -389,7 +389,7 @@ public boolean isCompatible(final List fields) { @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Database implements ToXContent { private static final ParseField PROVIDER_FIELD = new ParseField("provider"); - private static final ParseField MD5_HASH_FIELD = new ParseField("md5_hash"); + private static final ParseField SHA256_HASH_FIELD = new ParseField("sha256_hash"); private static final ParseField UPDATED_AT_FIELD = new ParseField("updated_at"); private static final ParseField UPDATED_AT_FIELD_READABLE = new ParseField("updated_at_field"); private static final ParseField FIELDS_FIELD = new ParseField("fields"); @@ -401,10 +401,10 @@ public static class Database implements ToXContent { */ private String provider; /** - * @param md5Hash MD5 hash value of a database file - * @return MD5 hash value of a database file + * @param sha256Hash SHA256 hash value of a database file + * @return SHA256 hash value of a database file */ - private String md5Hash; + private String sha256Hash; /** * @param updatedAt A date when the database was updated * @return A date when the database was updated @@ -426,16 +426,16 @@ public static class Database implements ToXContent { true, args -> { String provider = (String) args[0]; - String md5Hash = (String) args[1]; + String sha256Hash = (String) args[1]; Instant updatedAt = args[2] == null ? null : Instant.ofEpochMilli((Long) args[2]); Long validForInDays = (Long) args[3]; List fields = (List) args[4]; - return new Database(provider, md5Hash, updatedAt, validForInDays, fields); + return new Database(provider, sha256Hash, updatedAt, validForInDays, fields); } ); static { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PROVIDER_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MD5_HASH_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), SHA256_HASH_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), UPDATED_AT_FIELD); PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VALID_FOR_IN_DAYS_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), FIELDS_FIELD); @@ -447,8 +447,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (provider != null) { builder.field(PROVIDER_FIELD.getPreferredName(), provider); } - if (md5Hash != null) { - builder.field(MD5_HASH_FIELD.getPreferredName(), md5Hash); + if (sha256Hash != null) { + builder.field(SHA256_HASH_FIELD.getPreferredName(), sha256Hash); } if (updatedAt != null) { builder.timeField( diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 4fe3274c..07ea55c7 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -203,7 +203,7 @@ private String setupIndex(final DatasourceManifest manifest, final Datasource da * * Update is needed when all following conditions are met * 1. updatedAt value in datasource is equal or before updateAt value in manifest - * 2. MD5 hash value in datasource is different with MD5 hash value in manifest + * 2. SHA256 hash value in datasource is different with SHA256 hash value in manifest * * @param datasource * @param manifest @@ -214,7 +214,7 @@ private boolean shouldUpdate(final Datasource datasource, final DatasourceManife return false; } - if (manifest.getMd5Hash().equals(datasource.getDatabase().getMd5Hash())) { + if (manifest.getSha256Hash().equals(datasource.getDatabase().getSha256Hash())) { return false; } return true; diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index 11ea1861..1051693a 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -173,7 +173,7 @@ public Datasource randomDatasource() { .setFields(Arrays.asList(GeospatialTestHelper.randomLowerCaseString(), GeospatialTestHelper.randomLowerCaseString())); datasource.getDatabase().setProvider(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setUpdatedAt(now); - datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setValidForInDays(Randomness.get().nextInt(30) + 1l); datasource.getUpdateStats().setLastSkippedAt(now); datasource.getUpdateStats().setLastSucceededAt(now); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java index 4a61e612..786fc6f9 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/GeoIpDataFacadeTests.java @@ -111,7 +111,7 @@ public void testGetDatabaseReader() throws Exception { DatasourceManifest manifest = new DatasourceManifest( zipFile.toURI().toURL().toExternalForm(), "sample_valid.csv", - "fake_md5", + "fake_sha256", 1l, Instant.now().toEpochMilli(), "tester" @@ -128,7 +128,7 @@ public void testGetDatabaseReaderNoFile() throws Exception { DatasourceManifest manifest = new DatasourceManifest( zipFile.toURI().toURL().toExternalForm(), "no_file.csv", - "fake_md5", + "fake_sha256", 1l, Instant.now().toEpochMilli(), "tester" diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java index 9c7e30fc..d1a19c0c 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceTests.java @@ -36,7 +36,7 @@ public void testParser() throws Exception { datasource.getDatabase().setFields(Arrays.asList("field1", "field2")); datasource.getDatabase().setProvider("test_provider"); datasource.getDatabase().setUpdatedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); - datasource.getDatabase().setMd5Hash(GeospatialTestHelper.randomLowerCaseString()); + datasource.getDatabase().setSha256Hash(GeospatialTestHelper.randomLowerCaseString()); datasource.getDatabase().setValidForInDays(1l); datasource.getUpdateStats().setLastProcessingTimeInMillis(Randomness.get().nextLong()); datasource.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); @@ -56,7 +56,7 @@ public void testCurrentIndexName() { Datasource datasource = new Datasource(); datasource.setName(id); datasource.getDatabase().setProvider("provider"); - datasource.getDatabase().setMd5Hash("md5Hash"); + datasource.getDatabase().setSha256Hash("sha256Hash"); datasource.getDatabase().setUpdatedAt(now); datasource.getDatabase().setValidForInDays(10l); datasource.getDatabase().setFields(new ArrayList<>()); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 517c45a0..8d78f44b 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -46,7 +46,7 @@ public void testUpdateDatasourceSkip() throws Exception { datasource.setState(DatasourceState.AVAILABLE); datasource.getUpdateStats().setLastSkippedAt(null); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt())); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash()); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash()); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); // Run @@ -69,7 +69,7 @@ public void testUpdateDatasourceInvalidFile() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); @@ -87,7 +87,7 @@ public void testUpdateDatasourceIncompatibleFields() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name", "additional_field")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); @@ -105,7 +105,7 @@ public void testUpdateDatasource() throws Exception { Datasource datasource = new Datasource(); datasource.setState(DatasourceState.AVAILABLE); datasource.getDatabase().setUpdatedAt(Instant.ofEpochMilli(manifest.getUpdatedAt() - 1)); - datasource.getDatabase().setMd5Hash(manifest.getMd5Hash().substring(1)); + datasource.getDatabase().setSha256Hash(manifest.getSha256Hash().substring(1)); datasource.getDatabase().setFields(Arrays.asList("country_name")); datasource.setEndpoint(manifestFile.toURI().toURL().toExternalForm()); datasource.getUpdateStats().setLastSucceededAt(null); @@ -116,7 +116,7 @@ public void testUpdateDatasource() throws Exception { // Verify assertEquals(manifest.getProvider(), datasource.getDatabase().getProvider()); - assertEquals(manifest.getMd5Hash(), datasource.getDatabase().getMd5Hash()); + assertEquals(manifest.getSha256Hash(), datasource.getDatabase().getSha256Hash()); assertEquals(Instant.ofEpochMilli(manifest.getUpdatedAt()), datasource.getDatabase().getUpdatedAt()); assertEquals(manifest.getValidForInDays(), datasource.getDatabase().getValidForInDays()); assertNotNull(datasource.getUpdateStats().getLastSucceededAt()); diff --git a/src/test/resources/ip2geo/manifest.json b/src/test/resources/ip2geo/manifest.json index 652bc9d8..4986fbd8 100644 --- a/src/test/resources/ip2geo/manifest.json +++ b/src/test/resources/ip2geo/manifest.json @@ -1,7 +1,7 @@ { "url": "https://test.com/db.zip", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "sample_provider" diff --git a/src/test/resources/ip2geo/manifest_invalid_url.json b/src/test/resources/ip2geo/manifest_invalid_url.json index 77d68aaf..4e806f49 100644 --- a/src/test/resources/ip2geo/manifest_invalid_url.json +++ b/src/test/resources/ip2geo/manifest_invalid_url.json @@ -1,7 +1,7 @@ { "url": "invalid://test.com/db.zip", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "sample_provider" diff --git a/src/test/resources/ip2geo/manifest_template.json b/src/test/resources/ip2geo/manifest_template.json index 4c273fa4..92ceb590 100644 --- a/src/test/resources/ip2geo/manifest_template.json +++ b/src/test/resources/ip2geo/manifest_template.json @@ -1,7 +1,7 @@ { "url": "URL", "db_name": "sample_valid.csv", - "md5_hash": "safasdfaskkkesadfasdf", + "sha256_hash": "safasdfaskkkesadfasdf", "valid_for_in_days": 30, "updated_at": 3134012341236, "provider": "maxmind"