[#1503] feat(IT): Gravitino web e2e test framework (#1689)

### What changes were proposed in this pull request?

1. Automatic download `Chrome` & `Chrome Driver` into `./intergration-test/build/chrome` directory. need to download about 150 MB zip file for the first time.
2. Support embedded and deploy test mode.
3. Support Linux and macOS operation systems
4. Successed load metalake webpage and check the browser title.
5. #1687 


### Why are the changes needed?

Gravitino Web UI needs an E2E test to ensure Web UI usability.

Fix: #1503

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

CI Passed

---------

Co-authored-by: Qi Yu <[email protected]>

gradle/libs.versions.toml

@@ -43,6 +43,8 @@ jsqlparser = "4.2"
 mysql = "8.0.23"
 postgresql = "42.6.0"
 immutables-value = "2.10.0"
+selenium = "3.141.59"
+rauschig = "1.2.0"
 
 protobuf-plugin = "0.9.2"
 spotless-plugin = '6.11.0'
@@ -142,6 +144,8 @@ postgresql-driver = { group = "org.postgresql", name = "postgresql", version.ref
 minikdc = { group = "org.apache.hadoop", name = "hadoop-minikdc", version.ref = "hadoop-minikdc"}
 immutables-value = { module = "org.immutables:value", version.ref = "immutables-value" }
 commons-cli = { group = "commons-cli", name = "commons-cli", version.ref = "commons-cli" }
+selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version.ref = "selenium" }
+rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" }
 
 [bundles]
 log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"]

integration-test/build.gradle.kts

@@ -123,6 +123,8 @@ dependencies {
   testImplementation(libs.mysql.driver)
   testImplementation(libs.postgresql.driver)
   implementation(libs.commons.cli)
+  testImplementation(libs.selenium)
+  testImplementation(libs.rauschig)
 }
 
 /* Optimizing integration test execution conditions */
@@ -269,6 +271,7 @@ tasks.test {
 
       // Default use MiniGravitino to run integration tests
       environment("GRAVITINO_ROOT_DIR", rootDir.path)
+      environment("IT_PROJECT_DIR", buildDir.path)
       environment("HADOOP_USER_NAME", "datastrato")
       environment("HADOOP_HOME", "/tmp")
       environment("PROJECT_VERSION", version)
@@ -296,6 +299,7 @@ tasks.test {
       } else if (testMode == "embedded") {
         environment("GRAVITINO_HOME", rootDir.path)
         environment("GRAVITINO_TEST", "true")
+        environment("GRAVITINO_WAR", rootDir.path + "/web/dist/")
         systemProperty("testMode", "embedded")
       } else {
         throw GradleException("Gravitino integration tests only support [-PtestMode=embedded] or [-PtestMode=deploy] mode!")

integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/MetalakePageTest.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.web.ui;
+
+import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class MetalakePageTest extends AbstractWebIT {
+  @Test
+  public void homePage() {
+    String title = driver.getTitle();
+    Assertions.assertEquals("Gravitino", title);
+  }
+}

integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/utils/AbstractWebIT.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.web.ui.utils;
+
+import com.datastrato.gravitino.integration.test.util.AbstractIT;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.openqa.selenium.WebDriver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// AbstractWebIT provides a WebDriver instance for WEB UI tests.
+public class AbstractWebIT extends AbstractIT {
+  protected static final Logger LOG = LoggerFactory.getLogger(AbstractWebIT.class);
+  protected static WebDriver driver;
+  protected static final long MAX_IMPLICIT_WAIT = 30;
+
+  @BeforeAll
+  public static void startUp() {
+    driver = WebDriverManager.getWebDriver(getGravitinoServerPort());
+  }
+
+  @AfterAll
+  public static void tearDown() {
+    driver.quit();
+  }
+}

integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/utils/ChromeWebDriverProvider.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.web.ui.utils;
+
+import com.datastrato.gravitino.integration.test.util.ITUtils;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.rauschig.jarchivelib.ArchiveFormat;
+import org.rauschig.jarchivelib.Archiver;
+import org.rauschig.jarchivelib.ArchiverFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// ChromeWebDriverProvider provides a ChromeDriver instance for WEB UI tests.
+public class ChromeWebDriverProvider implements WebDriverProvider {
+  protected static final Logger LOG = LoggerFactory.getLogger(ChromeWebDriverProvider.class);
+  private final String chromeDriverBinName;
+  private final String chromeBinName;
+  private static final String downLoadDir =
+      ITUtils.joinPath(System.getenv("IT_PROJECT_DIR"), "chrome");
+
+  public ChromeWebDriverProvider() {
+    if (SystemUtils.IS_OS_MAC_OSX) {
+      this.chromeDriverBinName = ITUtils.joinPath("chromedriver_mac64", "chromedriver");
+      this.chromeBinName = ITUtils.joinPath("chrome-mac", "Chromium.app");
+    } else if (SystemUtils.IS_OS_LINUX) {
+      this.chromeDriverBinName = ITUtils.joinPath("chromedriver_linux64", "chromedriver");
+      this.chromeBinName = ITUtils.joinPath("chrome-linux", "chrome");
+    } else if (SystemUtils.IS_OS_WINDOWS) {
+      this.chromeDriverBinName = ITUtils.joinPath("chromedriver_win32", "chromedriver.exe");
+      this.chromeBinName = ITUtils.joinPath("chrome-win", "chrome.exe");
+    } else {
+      throw new RuntimeException("Unsupported OS : " + SystemUtils.OS_NAME);
+    }
+  }
+
+  @Override
+  public void downloadWebDriver() {
+    // Chrome release list in here:
+    // https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html
+    String chromeDownloadURL = "", chromeDriverDownloadURL = "";
+    String chromeZipFile = "", chromeDriverZipFile = "";
+    if (SystemUtils.IS_OS_LINUX) {
+      chromeZipFile = "chrome-linux.zip";
+      chromeDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1000022%2Fchrome-linux.zip?generation=1651778257041732&alt=media";
+
+      chromeDriverZipFile = "chromedriver_linux64.zip";
+      chromeDriverDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1000022%2Fchromedriver_linux64.zip?generation=1651778262235204&alt=media";
+    } else if (SystemUtils.IS_OS_MAC_OSX) {
+      chromeZipFile = "chrome-mac.zip";
+      chromeDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F1000022%2Fchrome-mac.zip?generation=1651779420087881&alt=media";
+
+      chromeDriverZipFile = "chromedriver_mac64.zip";
+      chromeDriverDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F1000022%2Fchromedriver_mac64.zip?generation=1651779426705083&alt=media";
+    } else if (SystemUtils.IS_OS_WINDOWS) {
+      chromeZipFile = "chrome-win.zip";
+      chromeDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Win_x64%2F1000027%2Fchrome-win.zip?generation=1651780728332948&alt=media";
+
+      chromeDriverZipFile = "chromedriver_win32.zip";
+      chromeDriverDownloadURL =
+          "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Win_x64%2F1000027%2Fchromedriver_win32.zip?generation=1651780916599219&alt=media";
+    }
+
+    downloadZipFile(chromeDriverDownloadURL, chromeDriverZipFile, chromeDriverBinName);
+    downloadZipFile(chromeDownloadURL, chromeZipFile, chromeBinName);
+
+    LOG.info("Download the chromeDriver to " + downLoadDir + " successfully.");
+  }
+
+  @Override
+  public WebDriver createWebDriver() {
+    System.setProperty(
+        "webdriver.chrome.driver", ITUtils.joinPath(downLoadDir, chromeDriverBinName));
+    ChromeOptions chromeOptions = new ChromeOptions();
+    if (SystemUtils.IS_OS_MAC_OSX) {
+      chromeOptions.setBinary(
+          ITUtils.joinPath(downLoadDir, chromeBinName, "Contents", "MacOS", "Chromium"));
+    } else {
+      chromeOptions.setBinary(ITUtils.joinPath(downLoadDir, chromeBinName));
+      chromeOptions.addArguments("--headless");
+    }
+
+    return new ChromeDriver(chromeOptions);
+  }
+
+  private void downloadZipFile(String url, String zipFileName, String fileName) {
+    File targetFile = new File(downLoadDir, fileName);
+    if (targetFile.exists()) {
+      LOG.info("The file " + targetFile.getAbsolutePath() + " already exists, skip download.");
+      return;
+    }
+
+    try {
+      LOG.info("Download the zip file from " + url + " to " + downLoadDir);
+      File chromeDriverZip = new File(ChromeWebDriverProvider.downLoadDir, zipFileName);
+      FileUtils.copyURLToFile(new URL(url), chromeDriverZip);
+
+      LOG.info("Extract the zip file from " + chromeDriverZip.getAbsolutePath());
+      Archiver archiver = ArchiverFactory.createArchiver(ArchiveFormat.ZIP);
+      archiver.extract(new File(downLoadDir, zipFileName), new File(downLoadDir));
+    } catch (IOException e) {
+      LOG.error(
+          "Download of: " + url + ", failed in path " + ChromeWebDriverProvider.downLoadDir, e);
+      throw new RuntimeException(e);
+    }
+    LOG.info("Download the zip file from " + url + " to " + downLoadDir + " successfully.");
+  }
+}

integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/utils/WebDriverManager.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.web.ui.utils;
+
+import java.util.concurrent.TimeUnit;
+import org.openqa.selenium.By;
+import org.openqa.selenium.Dimension;
+import org.openqa.selenium.TimeoutException;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.support.ui.ExpectedCondition;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// WebDriverManager manage the multiple web drivers to run the WEB UI tests.
+public class WebDriverManager {
+  public static final Logger LOG = LoggerFactory.getLogger(WebDriverManager.class);
+
+  public static WebDriver getWebDriver(int port) {
+    String url = String.format("http://127.0.0.1:%d", port);
+
+    WebDriverProvider provide = new ChromeWebDriverProvider();
+    WebDriver driver = generateWebDriver(provide);
+    driver.manage().timeouts().implicitlyWait(AbstractWebIT.MAX_IMPLICIT_WAIT, TimeUnit.SECONDS);
+    driver.get(url);
+
+    // wait for webpage load compiled.
+    long start = System.currentTimeMillis();
+    boolean loaded = false;
+    while (System.currentTimeMillis() - start < 60 * 1000) {
+      try {
+        (new WebDriverWait(driver, 10))
+            .until(
+                new ExpectedCondition<Boolean>() {
+                  @Override
+                  public Boolean apply(WebDriver d) {
+                    String gravitinoVersion = d.findElement(By.id("gravitino_version")).getText();
+                    String projectVersion = System.getenv("PROJECT_VERSION");
+                    return projectVersion.equalsIgnoreCase(gravitinoVersion);
+                  }
+                });
+        loaded = true;
+        break;
+      } catch (TimeoutException e) {
+        LOG.info("Exception in WebDriverManager while WebDriverWait ", e);
+        driver.navigate().to(url);
+      }
+    }
+
+    if (!loaded) {
+      throw new RuntimeException("Webpage not loaded in 60 seconds.");
+    }
+    Dimension d = new Dimension(1440, 1080);
+    driver.manage().window().setSize(d);
+
+    return driver;
+  }
+
+  private static WebDriver generateWebDriver(WebDriverProvider provide) {
+    provide.downloadWebDriver();
+    WebDriver driver = provide.createWebDriver();
+    return driver;
+  }
+}

integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/utils/WebDriverProvider.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.web.ui.utils;
+
+import org.openqa.selenium.WebDriver;
+
+// WebDriverProvider is an interface for WEB Driver provider.
+public interface WebDriverProvider {
+
+  /** Download the browser web driver. */
+  public void downloadWebDriver();
+
+  /** create a new browser web driver */
+  public WebDriver createWebDriver();
+}

web/lib/layout/VersionView.js

@@ -11,7 +11,11 @@ import { useAppSelector } from '@/lib/hooks/useStore'
 const VersionView = () => {
   const store = useAppSelector(state => state.sys)
 
-  return <Typography variant='subtitle2'>{store.version}</Typography>
+  return (
+    <Typography variant='subtitle2' id='gravitino_version'>
+      {store.version}
+    </Typography>
+  )
 }
 
 export default VersionView