From 3f9357ce2b19408972d776058df36aa202cd2a85 Mon Sep 17 00:00:00 2001 From: amakwana Date: Fri, 29 May 2020 01:15:45 -0400 Subject: [PATCH 01/10] sign & Verify model --- .../elide-dynamic-config-helpers/pom.xml | 5 + .../verify/DynamicConfigVerifier.java | 115 ++++++++++++++++++ .../verify/DynamicConfigVerifiesTest.java | 67 ++++++++++ 3 files changed, 187 insertions(+) create mode 100644 elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java create mode 100644 elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java diff --git a/elide-contrib/elide-dynamic-config-helpers/pom.xml b/elide-contrib/elide-dynamic-config-helpers/pom.xml index a55c1eb7f9..5beb8bcd86 100644 --- a/elide-contrib/elide-dynamic-config-helpers/pom.xml +++ b/elide-contrib/elide-dynamic-config-helpers/pom.xml @@ -69,6 +69,11 @@ commons-io ${commons-io.version} + + org.apache.commons + commons-compress + 1.20 + org.slf4j slf4j-api diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java new file mode 100644 index 0000000000..c454006a27 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -0,0 +1,115 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.verify; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + +import lombok.extern.slf4j.Slf4j; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.util.Base64; + +@Slf4j +public class DynamicConfigVerifier { + + /** + * Main Methode to Verify Signature of Model Tar file. + * @param args : expects 3 arguments. + * @throws IllegalStateException + */ + public static void main(String[] args) { + if (args == null || args.length != 3) { + usage(); + throw new IllegalStateException("No Arguments provided!"); + } + String modelTarFile = args[0]; + String signatureFile = args[1]; + String publicKeyName = args[2]; + + if (verify(tarContents(modelTarFile), signatureFile, getPublicKey(publicKeyName))) { + log.info("Successfully Validated " + modelTarFile); + } + else { + log.error("Could not verify " + modelTarFile + " with details provided"); + } + } + + private static PublicKey getPublicKey(String keyName) { + PublicKey publicKey = null; + try { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + Certificate cert = keyStore.getCertificate(keyName); + publicKey = cert.getPublicKey(); + } catch (KeyStoreException e) { + log.error(e.getMessage()); + throw new IllegalArgumentException("Key " + keyName + " is not availabe in keystore"); + } + return publicKey; + } + + /** + * Verify signature of tar.gz. + * @param fileList + * @param signature + * @param publicKey + * @return whether the file can be verified by given key & signature. + * @throws Exception + */ + public static boolean verify(String fileList, String signature, PublicKey publicKey) { + Signature publicSignature; + try { + publicSignature = Signature.getInstance("SHA256withRSA"); + publicSignature.initVerify(publicKey); + publicSignature.update(fileList.getBytes(StandardCharsets.UTF_8)); + byte[] signatureBytes = Base64.getDecoder().decode(signature); + return publicSignature.verify(signatureBytes); + } catch (NoSuchAlgorithmException e) { + log.error(e.getMessage()); + } catch (InvalidKeyException e) { + log.error(e.getMessage()); + } catch (SignatureException e) { + log.error(e.getMessage()); + } + return false; + } + + private static String tarContents(String archiveFile) { + StringBuffer sb = new StringBuffer(); + try (TarArchiveInputStream archive = new TarArchiveInputStream( + new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile))))) { + TarArchiveEntry entry; + while ((entry = archive.getNextTarEntry()) != null) { + sb.append(entry.getName()); + } + } catch (IOException e) { + log.error(e.getMessage()); + throw new IllegalArgumentException("archiveFile " + archiveFile + " is not available"); + } + return sb.toString(); + } + + private static void usage() { + log.info("\n Usage: java -cp " + + " com.yahoo.elide.contrib.dynamicconfighelpers.validator.DynamicConfigVerifier\n" + + " Date: Tue, 2 Jun 2020 03:46:37 -0400 Subject: [PATCH 02/10] review comments --- .../elide-dynamic-config-helpers/pom.xml | 3 +- .../verify/DynamicConfigVerifier.java | 164 +++++++++++------- .../verify/DynamicConfigVerifiesTest.java | 48 +++-- 3 files changed, 142 insertions(+), 73 deletions(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/pom.xml b/elide-contrib/elide-dynamic-config-helpers/pom.xml index 3d5668eb43..2e9c1dc28f 100644 --- a/elide-contrib/elide-dynamic-config-helpers/pom.xml +++ b/elide-contrib/elide-dynamic-config-helpers/pom.xml @@ -45,6 +45,7 @@ 1.3.0 2.6 1.4 + 1.20 @@ -78,7 +79,7 @@ org.apache.commons commons-compress - 1.20 + ${commons-compress.version} org.slf4j diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index c454006a27..0739b09ee5 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -5,14 +5,24 @@ */ package com.yahoo.elide.contrib.dynamicconfighelpers.verify; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; import lombok.extern.slf4j.Slf4j; import java.io.BufferedInputStream; +import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; @@ -29,20 +39,39 @@ public class DynamicConfigVerifier { /** - * Main Methode to Verify Signature of Model Tar file. + * Main Method to Verify Signature of Model Tar file. * @param args : expects 3 arguments. - * @throws IllegalStateException + * @throws ParseException + * @throws IOException + * @throws KeyStoreException + * @throws FileNotFoundException + * @throws SignatureException + * @throws NoSuchAlgorithmException + * @throws InvalidKeyException + * */ - public static void main(String[] args) { - if (args == null || args.length != 3) { - usage(); - throw new IllegalStateException("No Arguments provided!"); + public static void main(String[] args) throws ParseException, InvalidKeyException, NoSuchAlgorithmException, + SignatureException, FileNotFoundException, KeyStoreException, IOException { + + Options options = prepareOptions(); + CommandLine cli = new DefaultParser().parse(options, args); + + if (cli.hasOption("help")) { + printHelp(options); + return; } - String modelTarFile = args[0]; - String signatureFile = args[1]; - String publicKeyName = args[2]; + if (!cli.hasOption("tarFile") || !cli.hasOption("signatureFile") || !cli.hasOption("publicKeyName")) { + printHelp(options); + throw new MissingOptionException("Missing required option"); + } + + String modelTarFile = cli.getOptionValue("tarFile"); + String signatureFile = cli.getOptionValue("signatureFile"); + String publicKeyName = cli.getOptionValue("publicKeyName"); + long tarFileSize = getFileSize(modelTarFile); - if (verify(tarContents(modelTarFile), signatureFile, getPublicKey(publicKeyName))) { + if (verify(readTarContents(modelTarFile), tarFileSize, + signatureFile, getPublicKey(publicKeyName))) { log.info("Successfully Validated " + modelTarFile); } else { @@ -50,66 +79,81 @@ public static void main(String[] args) { } } - private static PublicKey getPublicKey(String keyName) { - PublicKey publicKey = null; - try { - KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - Certificate cert = keyStore.getCertificate(keyName); - publicKey = cert.getPublicKey(); - } catch (KeyStoreException e) { - log.error(e.getMessage()); - throw new IllegalArgumentException("Key " + keyName + " is not availabe in keystore"); - } - return publicKey; - } - /** * Verify signature of tar.gz. - * @param fileList - * @param signature - * @param publicKey - * @return whether the file can be verified by given key & signature. - * @throws Exception + * @param fileList : list of files + * @param sizeOfFile : size of tar file + * @param signature : file containing signature + * @param publicKey : public key name + * @return whether the file can be verified by given key and signature + * @throws NoSuchAlgorithmException + * @throws InvalidKeyException + * @throws SignatureException */ - public static boolean verify(String fileList, String signature, PublicKey publicKey) { + public static boolean verify(String fileList, long sizeOfFile, String signature, PublicKey publicKey) + throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { + Signature publicSignature; - try { - publicSignature = Signature.getInstance("SHA256withRSA"); - publicSignature.initVerify(publicKey); - publicSignature.update(fileList.getBytes(StandardCharsets.UTF_8)); - byte[] signatureBytes = Base64.getDecoder().decode(signature); - return publicSignature.verify(signatureBytes); - } catch (NoSuchAlgorithmException e) { - log.error(e.getMessage()); - } catch (InvalidKeyException e) { - log.error(e.getMessage()); - } catch (SignatureException e) { - log.error(e.getMessage()); - } - return false; + + publicSignature = Signature.getInstance("SHA256withRSA"); + publicSignature.initVerify(publicKey); + publicSignature.update((fileList + sizeOfFile).getBytes(StandardCharsets.UTF_8)); + byte[] signatureBytes = Base64.getDecoder().decode(signature); + return publicSignature.verify(signatureBytes); + } + + /** + * Get tar file size. + * @param modelTarFile + * @return size of tar file + */ + private static long getFileSize(String modelTarFile) { + return FileUtils.sizeOf(new File(modelTarFile)); + } + + /** + * Retrieve public key from Key Store. + * @param keyName : name of the public key + * @return publickey + */ + private static PublicKey getPublicKey(String keyName) throws KeyStoreException { + PublicKey publicKey = null; + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + Certificate cert = keyStore.getCertificate(keyName); + publicKey = cert.getPublicKey(); + return publicKey; } - private static String tarContents(String archiveFile) { + private static String readTarContents(String archiveFile) throws FileNotFoundException, IOException { StringBuffer sb = new StringBuffer(); - try (TarArchiveInputStream archive = new TarArchiveInputStream( - new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile))))) { - TarArchiveEntry entry; - while ((entry = archive.getNextTarEntry()) != null) { - sb.append(entry.getName()); - } - } catch (IOException e) { - log.error(e.getMessage()); - throw new IllegalArgumentException("archiveFile " + archiveFile + " is not available"); + TarArchiveInputStream archive = new TarArchiveInputStream( + new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); + TarArchiveEntry entry; + while ((entry = archive.getNextTarEntry()) != null) { + sb.append(entry.getName()); } return sb.toString(); } - private static void usage() { - log.info("\n Usage: java -cp " - + " com.yahoo.elide.contrib.dynamicconfighelpers.validator.DynamicConfigVerifier\n" - + " com.yahoo.elide.contrib.dynamicconfighelpers.verify.DynamicConfigVerifier", + options); } } diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java index 7d23c104e4..80afeb888c 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java @@ -5,9 +5,14 @@ */ package com.yahoo.elide.contrib.dynamicconfighelpers.verify; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.MissingOptionException; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; @@ -20,8 +25,8 @@ public class DynamicConfigVerifiesTest { - private KeyPair kp; - private String signature; + private static KeyPair kp; + private static String signature; private static KeyPair generateKeyPair() throws Exception { KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); @@ -40,28 +45,47 @@ private static String sign(String data, PrivateKey privateKey) throws Exception return Base64.getEncoder().encodeToString(signature); } - private void setup() throws Exception { + @BeforeAll + public static void setup() throws Exception { kp = generateKeyPair(); - signature = sign("testing-signature", kp.getPrivate()); + signature = sign("testing-signature5", kp.getPrivate()); } @Test public void testValidSignature() throws Exception { - setup(); - assertTrue(DynamicConfigVerifier.verify("testing-signature", signature, kp.getPublic())); + assertTrue(DynamicConfigVerifier.verify("testing-signature", 5, signature, kp.getPublic())); } @Test public void testInvalidSignature() throws Exception { - setup(); - assertFalse(DynamicConfigVerifier.verify("invalid-signature", signature, kp.getPublic())); + assertFalse(DynamicConfigVerifier.verify("invalid-signature", 5, signature, kp.getPublic())); } @Test - public void testFileSignature() throws Exception { - kp = generateKeyPair(); - signature = sign("testing-signature", kp.getPrivate()); + public void testHelpArgumnents() { + assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "-h" })); + assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "--help" })); + } + + @Test + public void testNoArgumnents() { + Exception e = assertThrows(MissingOptionException.class, () -> DynamicConfigVerifier.main(null)); + assertTrue(e.getMessage().startsWith("Missing required option")); + } - assertTrue(DynamicConfigVerifier.verify("testing-signature", signature, kp.getPublic())); + @Test + public void testOneEmptyArgumnents() { + Exception e = assertThrows(MissingOptionException.class, + () -> DynamicConfigVerifier.main(new String[] { "" })); + assertTrue(e.getMessage().startsWith("Missing required option")); + } + + @Test + public void testMissingArgumnentValue() { + Exception e = assertThrows(MissingArgumentException.class, + () -> DynamicConfigVerifier.main(new String[] { "--tarFile" })); + assertTrue(e.getMessage().startsWith("Missing argument for option")); + e = assertThrows(MissingArgumentException.class, () -> DynamicConfigVerifier.main(new String[] { "-t" })); + assertTrue(e.getMessage().startsWith("Missing argument for option")); } } From e80cffbb26f3044b7b0cb05e5f890253b888b5aa Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 2 Jun 2020 11:34:51 -0400 Subject: [PATCH 03/10] typo fix --- .../verify/DynamicConfigVerifiesTest.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java index 80afeb888c..22f2101bda 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java @@ -28,25 +28,8 @@ public class DynamicConfigVerifiesTest { private static KeyPair kp; private static String signature; - private static KeyPair generateKeyPair() throws Exception { - KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); - generator.initialize(2048, new SecureRandom()); - KeyPair pair = generator.generateKeyPair(); - return pair; - } - - private static String sign(String data, PrivateKey privateKey) throws Exception { - Signature privateSignature = Signature.getInstance("SHA256withRSA"); - privateSignature.initSign(privateKey); - privateSignature.update(data.getBytes(StandardCharsets.UTF_8)); - - byte[] signature = privateSignature.sign(); - - return Base64.getEncoder().encodeToString(signature); - } - @BeforeAll - public static void setup() throws Exception { + public static void setUp() throws Exception { kp = generateKeyPair(); signature = sign("testing-signature5", kp.getPrivate()); } @@ -62,30 +45,45 @@ public void testInvalidSignature() throws Exception { } @Test - public void testHelpArgumnents() { + public void testHelpArguments() { assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "-h" })); assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "--help" })); } @Test - public void testNoArgumnents() { + public void testNoArguments() { Exception e = assertThrows(MissingOptionException.class, () -> DynamicConfigVerifier.main(null)); assertTrue(e.getMessage().startsWith("Missing required option")); } @Test - public void testOneEmptyArgumnents() { + public void testOneEmptyArguments() { Exception e = assertThrows(MissingOptionException.class, () -> DynamicConfigVerifier.main(new String[] { "" })); assertTrue(e.getMessage().startsWith("Missing required option")); } @Test - public void testMissingArgumnentValue() { + public void testMissingArgumentValue() { Exception e = assertThrows(MissingArgumentException.class, () -> DynamicConfigVerifier.main(new String[] { "--tarFile" })); assertTrue(e.getMessage().startsWith("Missing argument for option")); e = assertThrows(MissingArgumentException.class, () -> DynamicConfigVerifier.main(new String[] { "-t" })); assertTrue(e.getMessage().startsWith("Missing argument for option")); } + + private static KeyPair generateKeyPair() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048, new SecureRandom()); + KeyPair pair = generator.generateKeyPair(); + return pair; + } + + private static String sign(String data, PrivateKey privateKey) throws Exception { + Signature privateSignature = Signature.getInstance("SHA256withRSA"); + privateSignature.initSign(privateKey); + privateSignature.update(data.getBytes(StandardCharsets.UTF_8)); + byte[] signature = privateSignature.sign(); + return Base64.getEncoder().encodeToString(signature); + } } From e6e6d3d8cc3a2c0a6a5c9382b3aa6046773bd735 Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 2 Jun 2020 23:53:21 -0400 Subject: [PATCH 04/10] indent fix --- elide-contrib/elide-dynamic-config-helpers/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/pom.xml b/elide-contrib/elide-dynamic-config-helpers/pom.xml index 2e9c1dc28f..335167227b 100644 --- a/elide-contrib/elide-dynamic-config-helpers/pom.xml +++ b/elide-contrib/elide-dynamic-config-helpers/pom.xml @@ -82,7 +82,7 @@ ${commons-compress.version} - org.slf4j + org.slf4j slf4j-api From 64685aedf257eb8bd7f77cc80d2784a97f3a2b16 Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 9 Jun 2020 06:04:19 -0400 Subject: [PATCH 05/10] review comment --- .../verify/DynamicConfigVerifier.java | 48 +++++++++---------- .../verify/DynamicConfigVerifiesTest.java | 6 +-- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index 0739b09ee5..53f94fb69b 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -35,6 +35,9 @@ import java.security.cert.Certificate; import java.util.Base64; +/** + * Util class to Verify model tar.gz file's RSA signature with available public key in key store. + */ @Slf4j public class DynamicConfigVerifier { @@ -48,10 +51,10 @@ public class DynamicConfigVerifier { * @throws SignatureException * @throws NoSuchAlgorithmException * @throws InvalidKeyException - * + * @throws MissingOptionException */ public static void main(String[] args) throws ParseException, InvalidKeyException, NoSuchAlgorithmException, - SignatureException, FileNotFoundException, KeyStoreException, IOException { + SignatureException, FileNotFoundException, KeyStoreException, IOException { Options options = prepareOptions(); CommandLine cli = new DefaultParser().parse(options, args); @@ -68,10 +71,8 @@ public static void main(String[] args) throws ParseException, InvalidKeyExceptio String modelTarFile = cli.getOptionValue("tarFile"); String signatureFile = cli.getOptionValue("signatureFile"); String publicKeyName = cli.getOptionValue("publicKeyName"); - long tarFileSize = getFileSize(modelTarFile); - if (verify(readTarContents(modelTarFile), tarFileSize, - signatureFile, getPublicKey(publicKeyName))) { + if (verify(readTarContents(modelTarFile), signatureFile, getPublicKey(publicKeyName))) { log.info("Successfully Validated " + modelTarFile); } else { @@ -81,8 +82,7 @@ signatureFile, getPublicKey(publicKeyName))) { /** * Verify signature of tar.gz. - * @param fileList : list of files - * @param sizeOfFile : size of tar file + * @param fileContent : content Of all config files * @param signature : file containing signature * @param publicKey : public key name * @return whether the file can be verified by given key and signature @@ -90,25 +90,34 @@ signatureFile, getPublicKey(publicKeyName))) { * @throws InvalidKeyException * @throws SignatureException */ - public static boolean verify(String fileList, long sizeOfFile, String signature, PublicKey publicKey) + public static boolean verify(String fileContent, String signature, PublicKey publicKey) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException { Signature publicSignature; publicSignature = Signature.getInstance("SHA256withRSA"); publicSignature.initVerify(publicKey); - publicSignature.update((fileList + sizeOfFile).getBytes(StandardCharsets.UTF_8)); + publicSignature.update(fileContent.getBytes(StandardCharsets.UTF_8)); byte[] signatureBytes = Base64.getDecoder().decode(signature); return publicSignature.verify(signatureBytes); } /** - * Get tar file size. - * @param modelTarFile - * @return size of tar file + * Read Content of all files. + * @param archiveFile : tar.gz file path + * @return appended content of all files in tar + * @throws FileNotFoundException + * @throws IOException */ - private static long getFileSize(String modelTarFile) { - return FileUtils.sizeOf(new File(modelTarFile)); + public static String readTarContents(String archiveFile) throws FileNotFoundException, IOException { + StringBuffer sb = new StringBuffer(); + TarArchiveInputStream archive = new TarArchiveInputStream( + new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); + TarArchiveEntry entry; + while ((entry = archive.getNextTarEntry()) != null) { + sb.append(FileUtils.readFileToString(new File(entry.getName()), StandardCharsets.UTF_8)); + } + return sb.toString(); } /** @@ -124,17 +133,6 @@ private static PublicKey getPublicKey(String keyName) throws KeyStoreException { return publicKey; } - private static String readTarContents(String archiveFile) throws FileNotFoundException, IOException { - StringBuffer sb = new StringBuffer(); - TarArchiveInputStream archive = new TarArchiveInputStream( - new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); - TarArchiveEntry entry; - while ((entry = archive.getNextTarEntry()) != null) { - sb.append(entry.getName()); - } - return sb.toString(); - } - /** * Define Arguments. */ diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java index 22f2101bda..a46645629c 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java @@ -31,17 +31,17 @@ public class DynamicConfigVerifiesTest { @BeforeAll public static void setUp() throws Exception { kp = generateKeyPair(); - signature = sign("testing-signature5", kp.getPrivate()); + signature = sign("testing-signature", kp.getPrivate()); } @Test public void testValidSignature() throws Exception { - assertTrue(DynamicConfigVerifier.verify("testing-signature", 5, signature, kp.getPublic())); + assertTrue(DynamicConfigVerifier.verify("testing-signature", signature, kp.getPublic())); } @Test public void testInvalidSignature() throws Exception { - assertFalse(DynamicConfigVerifier.verify("invalid-signature", 5, signature, kp.getPublic())); + assertFalse(DynamicConfigVerifier.verify("invalid-signature", signature, kp.getPublic())); } @Test From 9e4230d50d8d49ad62f2bc177ee9e0b2d6acaaf4 Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 9 Jun 2020 19:56:10 -0400 Subject: [PATCH 06/10] additional testcases --- .../verify/DynamicConfigVerifier.java | 29 +++++++--- .../verify/DynamicConfigVerifiesTest.java | 58 ++++++++++++++++++- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index 53f94fb69b..941b154fe1 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -15,15 +15,15 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; import lombok.extern.slf4j.Slf4j; import java.io.BufferedInputStream; -import java.io.File; +import java.io.BufferedReader; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; import java.security.KeyStore; @@ -54,7 +54,7 @@ public class DynamicConfigVerifier { * @throws MissingOptionException */ public static void main(String[] args) throws ParseException, InvalidKeyException, NoSuchAlgorithmException, - SignatureException, FileNotFoundException, KeyStoreException, IOException { + SignatureException, FileNotFoundException, KeyStoreException, IOException { Options options = prepareOptions(); CommandLine cli = new DefaultParser().parse(options, args); @@ -77,6 +77,7 @@ public static void main(String[] args) throws ParseException, InvalidKeyExceptio } else { log.error("Could not verify " + modelTarFile + " with details provided"); + System.exit(-1); } } @@ -111,12 +112,24 @@ public static boolean verify(String fileContent, String signature, PublicKey pub */ public static String readTarContents(String archiveFile) throws FileNotFoundException, IOException { StringBuffer sb = new StringBuffer(); - TarArchiveInputStream archive = new TarArchiveInputStream( - new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); - TarArchiveEntry entry; - while ((entry = archive.getNextTarEntry()) != null) { - sb.append(FileUtils.readFileToString(new File(entry.getName()), StandardCharsets.UTF_8)); + BufferedReader br = null; + TarArchiveInputStream archiveInputStream = null; + try { + archiveInputStream = new TarArchiveInputStream( + new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); + TarArchiveEntry entry; + while ((entry = archiveInputStream.getNextTarEntry()) != null) { + br = new BufferedReader(new InputStreamReader(archiveInputStream)); + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + } + } + } finally { + archiveInputStream.close(); + br.close(); } + return sb.toString(); } diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java index a46645629c..7db5bf12c0 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java @@ -12,9 +12,21 @@ import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.KeyPairGenerator; @@ -27,16 +39,25 @@ public class DynamicConfigVerifiesTest { private static KeyPair kp; private static String signature; + private static String tarContent = null; + private static final String TAR_FILE_PATH = "src/test/resources/test.tar.gz"; @BeforeAll public static void setUp() throws Exception { + createTarGZ(); kp = generateKeyPair(); - signature = sign("testing-signature", kp.getPrivate()); + tarContent = DynamicConfigVerifier.readTarContents(TAR_FILE_PATH); + signature = sign(tarContent, kp.getPrivate()); + } + + @AfterAll + public static void after() { + FileUtils.deleteQuietly(FileUtils.getFile(TAR_FILE_PATH)); } @Test public void testValidSignature() throws Exception { - assertTrue(DynamicConfigVerifier.verify("testing-signature", signature, kp.getPublic())); + assertTrue(DynamicConfigVerifier.verify(tarContent, signature, kp.getPublic())); } @Test @@ -86,4 +107,37 @@ private static String sign(String data, PrivateKey privateKey) throws Exception byte[] signature = privateSignature.sign(); return Base64.getEncoder().encodeToString(signature); } + + private static void createTarGZ() throws FileNotFoundException, IOException { + TarArchiveOutputStream tarOutputStream = null; + try { + String modelPath = "src/test/resources/models_missing/"; + tarOutputStream = new TarArchiveOutputStream(new GzipCompressorOutputStream( + new BufferedOutputStream(new FileOutputStream(new File(TAR_FILE_PATH))))); + addFileToTarGz(tarOutputStream, modelPath, ""); + } finally { + tarOutputStream.finish(); + tarOutputStream.close(); + } + } + + private static void addFileToTarGz(TarArchiveOutputStream tOut, String path, String base) throws IOException { + File f = new File(path); + String entryName = base + f.getName(); + TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName); + tOut.putArchiveEntry(tarEntry); + + if (f.isFile()) { + IOUtils.copy(new FileInputStream(f), tOut); + tOut.closeArchiveEntry(); + } else { + tOut.closeArchiveEntry(); + File[] children = f.listFiles(); + if (children != null) { + for (File child : children) { + addFileToTarGz(tOut, child.getAbsolutePath(), entryName + "/"); + } + } + } + } } From 34dd314dabe1d8517d97daafdd24c1f03fb67aa0 Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 9 Jun 2020 20:28:42 -0400 Subject: [PATCH 07/10] revert system exit --- .../dynamicconfighelpers/verify/DynamicConfigVerifier.java | 1 - 1 file changed, 1 deletion(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index 941b154fe1..dabfa67ada 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -77,7 +77,6 @@ public static void main(String[] args) throws ParseException, InvalidKeyExceptio } else { log.error("Could not verify " + modelTarFile + " with details provided"); - System.exit(-1); } } From 7b0f4b25a449872ed76835b6b8e085a3e6dedc64 Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 9 Jun 2020 20:39:49 -0400 Subject: [PATCH 08/10] fix codacy --- .../dynamicconfighelpers/verify/DynamicConfigVerifier.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index dabfa67ada..2b18aaf450 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -116,13 +116,14 @@ public static String readTarContents(String archiveFile) throws FileNotFoundExce try { archiveInputStream = new TarArchiveInputStream( new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile)))); - TarArchiveEntry entry; - while ((entry = archiveInputStream.getNextTarEntry()) != null) { + TarArchiveEntry entry = archiveInputStream.getNextTarEntry(); + while (entry != null) { br = new BufferedReader(new InputStreamReader(archiveInputStream)); String line; while ((line = br.readLine()) != null) { sb.append(line); } + entry = archiveInputStream.getNextTarEntry(); } } finally { archiveInputStream.close(); From 022a2a13e9de94d8064adabcd35d54c63f1c1f7c Mon Sep 17 00:00:00 2001 From: amakwana Date: Tue, 9 Jun 2020 20:44:51 -0400 Subject: [PATCH 09/10] add return --- .../dynamicconfighelpers/verify/DynamicConfigVerifier.java | 1 + 1 file changed, 1 insertion(+) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index 2b18aaf450..a80811cfdd 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -77,6 +77,7 @@ public static void main(String[] args) throws ParseException, InvalidKeyExceptio } else { log.error("Could not verify " + modelTarFile + " with details provided"); + return; } } From 5de1a66371bc1f4f26f44da0183e95682afb6acc Mon Sep 17 00:00:00 2001 From: amakwana Date: Wed, 10 Jun 2020 10:23:11 -0400 Subject: [PATCH 10/10] add system exit --- .../dynamicconfighelpers/verify/DynamicConfigVerifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java index a80811cfdd..17754797f8 100644 --- a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java @@ -77,7 +77,7 @@ public static void main(String[] args) throws ParseException, InvalidKeyExceptio } else { log.error("Could not verify " + modelTarFile + " with details provided"); - return; + System.exit(-1); } }