Elide dynamic config model verification (#1354)

* sign & Verify model

* review comments

* typo fix

* indent fix

* review comment

* additional testcases

* revert system exit

* fix codacy

* add return

* add system exit

Co-authored-by: amakwana <[email protected]>

elide-contrib/elide-dynamic-config-helpers/pom.xml

@@ -45,6 +45,7 @@
         <mdkt.compiler.version>1.3.0</mdkt.compiler.version>
         <commons-io.version>2.6</commons-io.version>
         <commons-cli.version>1.4</commons-cli.version>
+        <commons-compress.version>1.20</commons-compress.version>
     </properties>
 
     <dependencies>
@@ -75,6 +76,11 @@
             <artifactId>commons-io</artifactId>
             <version>${commons-io.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>${commons-compress.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifier.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2020, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.contrib.dynamicconfighelpers.verify;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingOptionException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.PublicKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.util.Base64;
+
+/**
+ * Util class to Verify model tar.gz file's RSA signature with available public key in key store.
+ */
+@Slf4j
+public class DynamicConfigVerifier {
+
+    /**
+     * Main Method to Verify Signature of Model Tar file.
+     * @param args : expects 3 arguments.
+     * @throws ParseException
+     * @throws IOException
+     * @throws KeyStoreException
+     * @throws FileNotFoundException
+     * @throws SignatureException
+     * @throws NoSuchAlgorithmException
+     * @throws InvalidKeyException
+     * @throws MissingOptionException
+     */
+    public static void main(String[] args) throws ParseException, InvalidKeyException, NoSuchAlgorithmException,
+    SignatureException, FileNotFoundException, KeyStoreException, IOException {
+
+        Options options = prepareOptions();
+        CommandLine cli = new DefaultParser().parse(options, args);
+
+        if (cli.hasOption("help")) {
+            printHelp(options);
+            return;
+        }
+        if (!cli.hasOption("tarFile") || !cli.hasOption("signatureFile") || !cli.hasOption("publicKeyName")) {
+            printHelp(options);
+            throw new MissingOptionException("Missing required option");
+        }
+
+        String modelTarFile = cli.getOptionValue("tarFile");
+        String signatureFile = cli.getOptionValue("signatureFile");
+        String publicKeyName = cli.getOptionValue("publicKeyName");
+
+        if (verify(readTarContents(modelTarFile), signatureFile, getPublicKey(publicKeyName))) {
+            log.info("Successfully Validated " + modelTarFile);
+        }
+        else {
+            log.error("Could not verify " + modelTarFile + " with details provided");
+            System.exit(-1);
+        }
+    }
+
+    /**
+     * Verify signature of tar.gz.
+     * @param fileContent : content Of all config files
+     * @param signature : file containing signature
+     * @param publicKey : public key name
+     * @return whether the file can be verified by given key and signature
+     * @throws NoSuchAlgorithmException
+     * @throws InvalidKeyException
+     * @throws SignatureException
+     */
+    public static boolean verify(String fileContent, String signature, PublicKey publicKey)
+            throws NoSuchAlgorithmException, InvalidKeyException, SignatureException {
+
+        Signature publicSignature;
+
+        publicSignature = Signature.getInstance("SHA256withRSA");
+        publicSignature.initVerify(publicKey);
+        publicSignature.update(fileContent.getBytes(StandardCharsets.UTF_8));
+        byte[] signatureBytes = Base64.getDecoder().decode(signature);
+        return publicSignature.verify(signatureBytes);
+    }
+
+    /**
+     * Read Content of all files.
+     * @param archiveFile : tar.gz file path
+     * @return appended content of all files in tar
+     * @throws FileNotFoundException
+     * @throws IOException
+     */
+    public static String readTarContents(String archiveFile) throws FileNotFoundException, IOException {
+        StringBuffer sb = new StringBuffer();
+        BufferedReader br = null;
+        TarArchiveInputStream archiveInputStream = null;
+        try {
+            archiveInputStream = new TarArchiveInputStream(
+                    new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archiveFile))));
+            TarArchiveEntry entry = archiveInputStream.getNextTarEntry();
+            while (entry  != null) {
+                br = new BufferedReader(new InputStreamReader(archiveInputStream));
+                String line;
+                while ((line = br.readLine()) != null) {
+                    sb.append(line);
+                }
+                entry = archiveInputStream.getNextTarEntry();
+            }
+        } finally {
+            archiveInputStream.close();
+            br.close();
+        }
+
+        return sb.toString();
+    }
+
+    /**
+     * Retrieve public key from Key Store.
+     * @param keyName : name of the public key
+     * @return publickey
+     */
+    private static PublicKey getPublicKey(String keyName) throws KeyStoreException {
+        PublicKey publicKey = null;
+        KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        Certificate cert = keyStore.getCertificate(keyName);
+        publicKey = cert.getPublicKey();
+        return publicKey;
+    }
+
+    /**
+     * Define Arguments.
+     */
+    private static final Options prepareOptions() {
+        Options options = new Options();
+        options.addOption(new Option("h", "help", false, "Print a help message and exit."));
+        options.addOption(new Option("t", "tarFile", true, "Path of the tar.gz file"));
+        options.addOption(new Option("s", "signatureFile", true, "Path of the file containing the signature"));
+        options.addOption(new Option("p", "publicKeyName", true, "Name of public key in keystore"));
+        return options;
+    }
+
+    /**
+     * Print Help.
+     */
+    private static void printHelp(Options options) {
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp(
+                "java -cp <Jar File> com.yahoo.elide.contrib.dynamicconfighelpers.verify.DynamicConfigVerifier",
+                options);
+    }
+}

elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/verify/DynamicConfigVerifiesTest.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2020, Yahoo Inc.
+ * Licensed under the Apache License, Version 2.0
+ * See LICENSE file in project root for terms.
+ */
+package com.yahoo.elide.contrib.dynamicconfighelpers.verify;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.MissingOptionException;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.PrivateKey;
+import java.security.SecureRandom;
+import java.security.Signature;
+import java.util.Base64;
+
+public class DynamicConfigVerifiesTest {
+
+    private static KeyPair kp;
+    private static String signature;
+    private static String tarContent = null;
+    private static final String TAR_FILE_PATH = "src/test/resources/test.tar.gz";
+
+    @BeforeAll
+    public static void setUp() throws Exception {
+        createTarGZ();
+        kp = generateKeyPair();
+        tarContent = DynamicConfigVerifier.readTarContents(TAR_FILE_PATH);
+        signature = sign(tarContent, kp.getPrivate());
+    }
+
+    @AfterAll
+    public static void after() {
+        FileUtils.deleteQuietly(FileUtils.getFile(TAR_FILE_PATH));
+    }
+
+    @Test
+    public void testValidSignature() throws Exception {
+        assertTrue(DynamicConfigVerifier.verify(tarContent, signature, kp.getPublic()));
+    }
+
+    @Test
+    public void testInvalidSignature() throws Exception {
+        assertFalse(DynamicConfigVerifier.verify("invalid-signature", signature, kp.getPublic()));
+    }
+
+    @Test
+    public void testHelpArguments() {
+        assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "-h" }));
+        assertDoesNotThrow(() -> DynamicConfigVerifier.main(new String[] { "--help" }));
+    }
+
+    @Test
+    public void testNoArguments() {
+        Exception e = assertThrows(MissingOptionException.class, () -> DynamicConfigVerifier.main(null));
+        assertTrue(e.getMessage().startsWith("Missing required option"));
+    }
+
+    @Test
+    public void testOneEmptyArguments() {
+        Exception e = assertThrows(MissingOptionException.class,
+                () -> DynamicConfigVerifier.main(new String[] { "" }));
+        assertTrue(e.getMessage().startsWith("Missing required option"));
+    }
+
+    @Test
+    public void testMissingArgumentValue() {
+        Exception e = assertThrows(MissingArgumentException.class,
+                () -> DynamicConfigVerifier.main(new String[] { "--tarFile" }));
+        assertTrue(e.getMessage().startsWith("Missing argument for option"));
+        e = assertThrows(MissingArgumentException.class, () -> DynamicConfigVerifier.main(new String[] { "-t" }));
+        assertTrue(e.getMessage().startsWith("Missing argument for option"));
+    }
+
+    private static KeyPair generateKeyPair() throws Exception {
+        KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
+        generator.initialize(2048, new SecureRandom());
+        KeyPair pair = generator.generateKeyPair();
+        return pair;
+    }
+
+    private static String sign(String data, PrivateKey privateKey) throws Exception {
+        Signature privateSignature = Signature.getInstance("SHA256withRSA");
+        privateSignature.initSign(privateKey);
+        privateSignature.update(data.getBytes(StandardCharsets.UTF_8));
+        byte[] signature = privateSignature.sign();
+        return Base64.getEncoder().encodeToString(signature);
+    }
+
+    private static void createTarGZ() throws FileNotFoundException, IOException {
+        TarArchiveOutputStream tarOutputStream = null;
+        try {
+            String modelPath  = "src/test/resources/models_missing/";
+            tarOutputStream = new TarArchiveOutputStream(new GzipCompressorOutputStream(
+                    new BufferedOutputStream(new FileOutputStream(new File(TAR_FILE_PATH)))));
+            addFileToTarGz(tarOutputStream, modelPath, "");
+        } finally {
+            tarOutputStream.finish();
+            tarOutputStream.close();
+        }
+    }
+
+    private static void addFileToTarGz(TarArchiveOutputStream tOut, String path, String base) throws IOException {
+        File f = new File(path);
+        String entryName = base + f.getName();
+        TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName);
+        tOut.putArchiveEntry(tarEntry);
+
+        if (f.isFile()) {
+            IOUtils.copy(new FileInputStream(f), tOut);
+            tOut.closeArchiveEntry();
+        } else {
+            tOut.closeArchiveEntry();
+            File[] children = f.listFiles();
+            if (children != null) {
+                for (File child : children) {
+                    addFileToTarGz(tOut, child.getAbsolutePath(), entryName + "/");
+                }
+            }
+        }
+    }
+}