[#2839] improvment(CI): add log of testcontainers (#2843)

### What changes were proposed in this pull request?

Separate TestContainers logs into a different log file.

### Why are the changes needed?

Fix: #2839 

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

by hand

.github/workflows/backend-integration-test.yml

@@ -104,7 +104,7 @@ jobs:
           name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}
           path: |
             build/reports
-            integration-test/build/integration-test-integration-test.log
+            integration-test/build/*.log
             integration-test/build/*.tar
             distribution/package/logs/gravitino-server.out
             distribution/package/logs/gravitino-server.log

catalogs/catalog-hadoop/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

catalogs/catalog-hive/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

catalogs/catalog-jdbc-mysql/src/test/resources/log4j2.properties

@@ -31,3 +31,26 @@ rootLogger.level = info
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

catalogs/catalog-jdbc-postgresql/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

catalogs/catalog-lakehouse-iceberg/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off

integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java

@@ -228,7 +228,7 @@ public static void startIntegrationTest() throws Exception {
 
   @AfterAll
   public static void stopIntegrationTest() throws IOException, InterruptedException {
-    if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) {
+    if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE) && miniGravitino != null) {
       miniGravitino.stop();
     } else {
       GravitinoITUtils.stopGravitinoServer();

integration-test/src/test/resources/log4j2.properties

@@ -23,11 +23,37 @@ appender.file.type = File
 appender.file.name = fileLogger
 appender.file.fileName = ${logPath}
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
 
 # Root logger level
 rootLogger.level = info
 
 # Root logger referring to console and file appenders
 rootLogger.appenderRef.stdout.ref = consoleLogger
 rootLogger.appenderRef.file.ref = fileLogger
+
+# File appender configuration for testcontainers
+appender.testcontainersFile.type = File
+appender.testcontainersFile.name = testcontainersLogger
+appender.testcontainersFile.fileName = build/testcontainers.log
+appender.testcontainersFile.layout.type = PatternLayout
+appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n
+
+# Logger for testcontainers
+logger.testcontainers.name = org.testcontainers
+logger.testcontainers.level = debug
+logger.testcontainers.additivity = false
+logger.testcontainers.appenderRef.file.ref = testcontainersLogger
+
+logger.tc.name = tc
+logger.tc.level = debug
+logger.tc.additivity = false
+logger.tc.appenderRef.file.ref = testcontainersLogger
+
+logger.docker.name = com.github.dockerjava
+logger.docker.level = warn
+logger.docker.additivity = false
+logger.docker.appenderRef.file.ref = testcontainersLogger
+
+logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire
+logger.http.level = off