replacing wix-embedded-mysql with testcontainers-java (#2384)

Signed-off-by: Abhijeet V <[email protected]>

.gitignore

@@ -14,6 +14,7 @@ servers/zms/zms_logs/
 servers/zms/zms_root/
 servers/zms/bin/
 servers/zms/schema/zms_server.mwb.bak
+servers/zms/src/test/resources/mysql/zms_server.sql
 servers/zts/zts_logs/
 servers/zts/zts_store/
 servers/zts/bin/

pom.xml

@@ -91,6 +91,7 @@
     <jersey.version>3.1.3</jersey.version>
     <jetty.version>11.0.17</jetty.version>
     <jjwt.version>0.11.5</jjwt.version>
+    <jna.version>5.13.0</jna.version>
     <logback.client.version>1.2.12</logback.client.version>
     <logback.server.version>1.4.11</logback.server.version>
     <maven-assembly-plugin.version>3.6.0</maven-assembly-plugin.version>
@@ -115,6 +116,7 @@
     <rdl.version>1.5.4</rdl.version>
     <slf4j.client.version>1.7.36</slf4j.client.version>
     <slf4j.server.version>2.0.9</slf4j.server.version>
+    <testcontainers.version>1.19.1</testcontainers.version>
     <testng.version>7.8.0</testng.version>
     <swagger.version>2.2.17</swagger.version>
     <commons.logging.version>1.2</commons.logging.version>

screwdriver.yaml

@@ -2,12 +2,15 @@
 ---
 
 shared:
-  image: openjdk:11
+  image: eclipse-temurin:11-jdk-focal
   annotations:
     screwdriver.cd/cpu: 8
     screwdriver.cd/ram: 16
     screwdriver.cd/disk: HIGH
     screwdriver.cd/timeout: 120
+    screwdriver.cd/dockerEnabled: true
+    screwdriver.cd/dockerCpu: TURBO
+    screwdriver.cd/dockerRam: HIGH
   environment:
     USER_SHELL_BIN: /bin/bash
 

screwdriver/scripts/build.sh

@@ -3,4 +3,6 @@
 set -e
 
 export PATH=$PATH:/usr/local/go/bin
+cp ${SD_SOURCE_DIR}/servers/zms/schema/zms_server.sql ${SD_SOURCE_DIR}/servers/zms/src/test/resources/mysql
+ls -ltr ${SD_SOURCE_DIR}/servers/zms/src/test/resources/mysql
 mvn -B install

screwdriver/scripts/install_deps.sh

@@ -13,8 +13,14 @@ echo "-----------------Install maven: -----------------"
 apt-get install -y maven
 
 echo "-----------------Install nodejs: -----------------"
-curl -sL https://deb.nodesource.com/setup_18.x | bash -
-apt-get install -y nodejs
+apt-get update
+apt-get install -y ca-certificates curl gnupg apt-transport-https
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+NODE_MAJOR=18
+echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+apt-get update
+apt-get install nodejs -y
 aptitude install -y npm
 npm install -g npm@latest
 
@@ -29,6 +35,18 @@ wget https://golang.org/dl/go1.19.linux-amd64.tar.gz
 tar -C /usr/local -xzf go1.19.linux-amd64.tar.gz
 export PATH=$PATH:/usr/local/go/bin
 
+echo "-----------------Install Docker: -----------------"
+# Add Docker's official GPG key:
+apt-get update
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+apt-key fingerprint 0EBFCD88
+
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable"
+apt-get update
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+docker system info
+ls -la $SD_DIND_SHARE_PATH
+
 # check all installed dependencies
 echo "-----------------Java Version: -----------------"
 java -version
@@ -40,3 +58,5 @@ echo "-----------------NPM Version: -----------------"
 npm -v
 echo "-----------------Golang Version: -----------------"
 go version
+echo "-----------------Docker Version: -----------------"
+docker version

servers/zms/pom.xml

@@ -11,7 +11,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -176,12 +176,6 @@
       <artifactId>swagger-jaxrs2-servlet-initializer-jakarta</artifactId>
       <version>${swagger.version}</version>
     </dependency>
-    <dependency>
-      <groupId>com.wix</groupId>
-      <artifactId>wix-embedded-mysql</artifactId>
-      <version>4.6.2</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>com.nimbusds</groupId>
       <artifactId>nimbus-jose-jwt</artifactId>
@@ -196,6 +190,22 @@
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>dynamodb-enhanced</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>mysql</artifactId>
+      <version>${testcontainers.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- JNA provides Java programs easy access to native shared libraries without writing anything but Java code.
+         This dependency is required for the testcontainers when running multiple tests in parallel
+         to look for local container runtime environment.
+    -->
+    <dependency>
+      <groupId>net.java.dev.jna</groupId>
+      <artifactId>jna</artifactId>
+      <version>${jna.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java

@@ -19,7 +19,6 @@
 import ch.qos.logback.classic.spi.LoggingEvent;
 import ch.qos.logback.core.AppenderBase;
 import com.google.common.io.Resources;
-import com.wix.mysql.EmbeddedMysql;
 import com.yahoo.athenz.auth.Authority;
 import com.yahoo.athenz.auth.Principal;
 import com.yahoo.athenz.auth.impl.FilePrivateKeyStore;
@@ -46,6 +45,7 @@
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.MySQLContainer;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -76,7 +76,7 @@ public class DBServiceTest {
     private String pubKeyK1         = null;
     private String pubKeyK2         = null;
     private final String auditRef   = "audittest";
-    private EmbeddedMysql mysqld;
+    private MySQLContainer<?> mysqld;
 
     // typically used when creating and deleting domains with all the tests
     //
@@ -111,7 +111,7 @@ public void setup() throws Exception {
         System.setProperty(ZMSConsts.ZMS_PROP_DOMAIN_ADMIN, "user.testadminuser");
 
         System.setProperty(ZMSConsts.ZMS_PROP_OBJECT_STORE_FACTORY_CLASS, "com.yahoo.athenz.zms.store.impl.JDBCObjectStoreFactory");
-        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, "jdbc:mysql://localhost:3310/zms_server");
+        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, mysqld.getJdbcUrl());
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_USER, DB_USER);
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_PASSWORD, DB_PASS);
 

servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSBinderTest.java

@@ -1,7 +1,7 @@
 package com.yahoo.athenz.zms;
 
-import com.wix.mysql.EmbeddedMysql;
 import com.yahoo.athenz.auth.impl.FilePrivateKeyStore;
+import org.testcontainers.containers.MySQLContainer;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -11,14 +11,14 @@ public class ZMSBinderTest {
     private static final String DB_USER = "admin";
     private static final String DB_PASS = "unit-test";
 
-    private EmbeddedMysql mysqld;
+    private MySQLContainer<?> mysqld;
 
     @BeforeClass
     public void setUp() {
         mysqld = ZMSTestUtils.startMemoryMySQL(DB_USER, DB_PASS);
 
         System.setProperty(ZMSConsts.ZMS_PROP_OBJECT_STORE_FACTORY_CLASS, "com.yahoo.athenz.zms.store.impl.JDBCObjectStoreFactory");
-        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, "jdbc:mysql://localhost:3310/zms_server");
+        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, mysqld.getJdbcUrl());
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_USER, DB_USER);
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_PASSWORD, DB_PASS);
         System.setProperty(ZMSConsts.ZMS_PROP_PRINCIPAL_STATE_UPDATER_DISABLE_TIMER, "true");

servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSTestInitializer.java

@@ -19,7 +19,6 @@
 package com.yahoo.athenz.zms;
 
 import com.google.common.io.Resources;
-import com.wix.mysql.EmbeddedMysql;
 import com.yahoo.athenz.auth.Authority;
 import com.yahoo.athenz.auth.Principal;
 import com.yahoo.athenz.auth.impl.FilePrivateKeyStore;
@@ -37,6 +36,8 @@
 
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
+import org.testcontainers.containers.MySQLContainer;
+
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -91,7 +92,7 @@ public class ZMSTestInitializer {
     public static synchronized int getRandomProductId() {
         return BASE_PRODUCT_ID + domainProductId.nextInt(99999999);
     }
-    private EmbeddedMysql mysqld;
+    private MySQLContainer<?> mysqld;
 
     public void startMemoryMySQL() {
         mysqld = ZMSTestUtils.startMemoryMySQL(DB_USER, DB_PASS);
@@ -103,14 +104,18 @@ public void stopMemoryMySQL() {
 
     public void setDatabaseReadOnlyMode(boolean readOnlyMode) {
         zms.dbService.defaultRetryCount = 3;
-        ZMSTestUtils.setDatabaseReadOnlyMode(mysqld, readOnlyMode);
+        try {
+            ZMSTestUtils.setDatabaseReadOnlyMode(mysqld, readOnlyMode, DB_USER, DB_PASS);
+        } catch (IOException | InterruptedException e) {
+            fail();
+        }
     }
 
     public void setUp() throws Exception {
         MockitoAnnotations.openMocks(this);
 
         System.setProperty(ZMSConsts.ZMS_PROP_OBJECT_STORE_FACTORY_CLASS, "com.yahoo.athenz.zms.store.impl.JDBCObjectStoreFactory");
-        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, "jdbc:mysql://localhost:3310/zms_server");
+        System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE, mysqld.getJdbcUrl());
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_USER, DB_USER);
         System.setProperty(ZMSConsts.ZMS_PROP_JDBC_RW_PASSWORD, DB_PASS);
 

servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSTestUtils.java

@@ -15,48 +15,72 @@
  */
 package com.yahoo.athenz.zms;
 
-import com.wix.mysql.EmbeddedMysql;
-import com.wix.mysql.Sources;
-import com.wix.mysql.config.MysqldConfig;
 import com.yahoo.rdl.Timestamp;
 import com.yahoo.rdl.UUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.MySQLContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.time.Duration;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
-import static com.wix.mysql.EmbeddedMysql.anEmbeddedMysql;
-import static com.wix.mysql.ScriptResolver.classPathScript;
-import static com.wix.mysql.config.MysqldConfig.aMysqldConfig;
-import static com.wix.mysql.distribution.Version.v5_7_latest;
-
 public class ZMSTestUtils {
 
-    public static EmbeddedMysql startMemoryMySQL(final String userName, final String password) {
+    private static final Logger LOG = LoggerFactory.getLogger(ZMSTestUtils.class);
+
+    public static MySQLContainer startMemoryMySQL(final String userName, final String password) {
 
-        System.out.println("Starting Embedded MySQL server...");
+        System.out.println("Starting MySQL server using testcontainers...");
+        MySQLContainer<?> mysql = null;
+        try {
 
-        final MysqldConfig config = aMysqldConfig(v5_7_latest)
-                .withPort(3310)
-                .withUser(userName, password)
-                .build();
+            String externalInitScriptPath = "schema/zms_server.sql";
+            File destinationInitScript = new File("src/test/resources/mysql/zms_server.sql");
+            FileUtils.copyFile(new File(externalInitScriptPath), destinationInitScript);
+            LOG.info("Copied {} to {}", externalInitScriptPath, destinationInitScript.getAbsolutePath());
+
+            mysql = new MySQLContainer<>(DockerImageName.parse("mysql/mysql-server:5.7").asCompatibleSubstituteFor("mysql"))
+                    .withDatabaseName("zms_server")
+                    .withUsername(userName)
+                    .withPassword(password)
+                    .withEnv("MYSQL_ROOT_PASSWORD", password)
+                    .withInitScript("mysql/zms_server.sql")
+                    .withStartupTimeout(Duration.ofMinutes(1))
+                    .withCopyFileToContainer(
+                            MountableFile.forClasspathResource("mysql/"),
+                            "/athenz-mysql-scripts/");
+            mysql.start();
+            mysql.followOutput(new Slf4jLogConsumer(LOG));
+        } catch (Throwable t) {
+            LOG.error("Unable to start MySQL server using testcontainers", t);
+        }
 
-        File sqlSchemaFile = new File("schema/zms_server.sql");
-        return anEmbeddedMysql(config)
-                .addSchema("zms_server", Sources.fromFile(sqlSchemaFile))
-                .start();
+        return mysql;
     }
 
-    public static void stopMemoryMySQL(EmbeddedMysql mysqld) {
-        System.out.println("Stopping Embedded MySQL server...");
+    public static void stopMemoryMySQL(MySQLContainer<?> mysqld) {
+        System.out.println("Stopping testcontainers MySQL server...");
         mysqld.stop();
     }
 
-    public static void setDatabaseReadOnlyMode(EmbeddedMysql mysqld, boolean readOnly) {
-        final String scriptName = readOnly ? "mysql/set-read-only.sql" : "mysql/unset-read-only.sql";
-        mysqld.executeScripts("zms_server", classPathScript(scriptName));
+    public static void setDatabaseReadOnlyMode(MySQLContainer<?> mysqld, boolean readOnly, final String username, final String password) throws IOException, InterruptedException {
+        final String scriptName = readOnly ? "set-read-only.sql" : "unset-read-only.sql";
+        try {
+            mysqld.execInContainer("mysql", "-u", "root", "-p"+password, "zms_server", "-e", "source /athenz-mysql-scripts/" + scriptName);
+        } catch (Throwable t) {
+            LOG.error("Unable to execute script in testcontainers mysql container: " + scriptName, t);
+        }
+
     }
 
     public static boolean verifyDomainRoleMember(DomainRoleMember domainRoleMember, MemberRole memberRole) {

servers/zms/src/test/resources/logback.xml

@@ -49,7 +49,10 @@
   <logger name="AuditSoxLogger" level="INFO" additivity="false">
     <appender-ref ref="AUDIT" />
   </logger>
-
+  <logger name="org.testcontainers" level="INFO"/>
+  <logger name="tc" level="INFO"/>
+  <logger name="com.github.dockerjava" level="WARN"/>
+  <logger name="com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire" level="OFF"/>
   <root>
     <level value="DEBUG" />
     <appender-ref ref="STDOUT" />

servers/zms/src/test/resources/sd_logback.xml

@@ -7,6 +7,11 @@
       <pattern>%-4relative [%thread] %-5level %class - %msg%n</pattern>
     </encoder>
   </appender>
+
+  <logger name="org.testcontainers" level="INFO"/>
+  <logger name="tc" level="INFO"/>
+  <logger name="com.github.dockerjava" level="WARN"/>
+  <logger name="com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire" level="OFF"/>
 
   <root>
     <level value="DEBUG" />