improve log

apache · Nov 22, 2022 · c5592d3 · c5592d3
1 parent dbbca5c
commit c5592d3
Show file tree

Hide file tree

Showing 7 changed files with 77 additions and 50 deletions.
diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
@@ -2,9 +2,9 @@
 
 import java.io.Closeable;
 
-import org.slf4j.Logger;
-
 import org.apache.hugegraph.util.Log;
+import org.apache.logging.log4j.LogManager;
+import org.slf4j.Logger;
 
 public class ShutdownHook {
 
@@ -23,6 +23,8 @@ public boolean hook(Closeable hook) {
             } catch (Throwable e) {
                 LOG.warn("Failed to execute shutdown hook: {}",
                           e.getMessage(), e);
+            } finally {
+                LogManager.shutdown();
             }
         });
         Runtime.getRuntime().addShutdownHook(this.threadShutdownHook);

diff --git a/computer-core/src/main/resources/log4j2.xml b/computer-core/src/main/resources/log4j2.xml
@@ -1,26 +1,51 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<configuration status="error">
+
+<!-- Config will be auto loaded every 60s -->
+<configuration status="error" monitorInterval="60">
+    <properties>
+        <property name="LOG_PATH">logs</property>
+        <property name="FILE_NAME">hugegraph-computer</property>
+    </properties>
+
     <appenders>
+        <!-- Container standard/console log -->
         <Console name="console" target="SYSTEM_OUT">
             <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
         </Console>
 
-        <RollingFile name="file" fileName="logs/hugegraph-computer.log"
-                     filePattern="logs/$${date:yyyy-MM}/hugegraph-computer-%d{yyyy-MM-dd}-%i.log">
-            <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
-            <SizeBasedTriggeringPolicy size="100MB"/>
-        </RollingFile>
+        <!-- Container file log -->
+        <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
+                                 filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
+                                 immediateFlush="false">
+            <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+            <!-- Trigger after exceeding 1day or 50MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 5 files per day & auto delete after over 1GB or 50 files -->
+            <DefaultRolloverStrategy max="5">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.log"/>
+                    <!-- Limit log amount & size -->
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="1GB" />
+                        <IfAccumulatedFileCount exceeds="50" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
+        </RollingRandomAccessFile>
     </appenders>
     <loggers>
         <root level="INFO">
             <appender-ref ref="console"/>
             <appender-ref ref="file"/>
         </root>
-        <logger name="com.baidu.hugegraph" level="INFO" additivity="false">
+        <Logger name="com.baidu.hugegraph" level="INFO" additivity="false">
             <appender-ref ref="console"/>
             <appender-ref ref="file"/>
-        </logger>
+        </Logger>
     </loggers>
-</configuration>
+</configuration>
diff --git a/...-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/OperatorEntrypoint.java b/...-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/OperatorEntrypoint.java
@@ -39,17 +39,18 @@
 import org.apache.commons.configuration2.MapConfiguration;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpStatus;
+import org.apache.hugegraph.config.HugeConfig;
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.util.ExecutorUtil;
+import org.apache.hugegraph.util.Log;
+import org.apache.logging.log4j.LogManager;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.computer.k8s.Constants;
 import com.baidu.hugegraph.computer.k8s.operator.common.AbstractController;
 import com.baidu.hugegraph.computer.k8s.operator.config.OperatorOptions;
 import com.baidu.hugegraph.computer.k8s.operator.controller.ComputerJobController;
 import com.baidu.hugegraph.computer.k8s.util.KubeUtil;
-import org.apache.hugegraph.config.HugeConfig;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.util.ExecutorUtil;
-import org.apache.hugegraph.util.Log;
 import com.sun.net.httpserver.HttpServer;
 
 import io.fabric8.kubernetes.api.model.ConfigMap;
@@ -182,6 +183,8 @@ public synchronized void shutdown() {
             this.httpServer.stop(0);
             this.httpServer = null;
         }
+
+        LogManager.shutdown();
     }
 
     private HugeConfig configFromSysPropsOrEnvVars() {

diff --git a/computer-k8s-operator/src/main/resources/log4j2.xml b/computer-k8s-operator/src/main/resources/log4j2.xml
@@ -9,13 +9,13 @@
     <appenders>
         <Console name="console" target="SYSTEM_OUT">
             <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
         </Console>
 
         <RollingFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
                      filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log">
             <ThresholdFilter level="${env:LOG_LEVEL:-INFO}" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
             <SizeBasedTriggeringPolicy size="100MB"/>
         </RollingFile>
     </appenders>

diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
@@ -6,13 +6,14 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.hugegraph.testutil.Assert;
+import org.apache.hugegraph.util.Log;
 import org.junit.Test;
 import org.slf4j.Logger;
 
-import org.apache.hugegraph.testutil.Assert;
-import org.apache.hugegraph.util.Log;
+import com.baidu.hugegraph.computer.suite.unit.UnitTestBase;
 
-public class HugeGraphComputerTest {
+public class HugeGraphComputerTest extends UnitTestBase {
 
     private static final Logger LOG = Log.logger(HugeGraphComputerTest.class);
 
@@ -84,16 +85,4 @@ public void testPrintUncaughtException() throws InterruptedException {
         t.join();
         Assert.assertTrue(isRun.get());
     }
-
-    private boolean existError(Throwable[] exceptions) {
-        boolean error = false;
-        for (Throwable e : exceptions) {
-            if (e != null) {
-                error = true;
-                LOG.warn("There exist error:", e);
-                break;
-            }
-        }
-        return error;
-    }
 }
diff --git a/...r-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/IntegrateTestSuite.java b/...r-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/IntegrateTestSuite.java
@@ -19,15 +19,16 @@
 
 package com.baidu.hugegraph.computer.suite.integrate;
 
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.testutil.Whitebox;
+import org.apache.hugegraph.util.Log;
+import org.apache.logging.log4j.LogManager;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.computer.core.config.ComputerOptions;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.testutil.Whitebox;
-import org.apache.hugegraph.util.Log;
 
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
@@ -39,6 +40,8 @@ public class IntegrateTestSuite {
 
     @BeforeClass
     public static void setup() {
+        Runtime.getRuntime().addShutdownHook(new Thread(LogManager::shutdown));
+
         LOG.info("Setup for IntegrateTestSuite of hugegraph-computer");
 
         // Don't forget to register options

diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestBase.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestBase.java
@@ -19,6 +19,22 @@
 
 package com.baidu.hugegraph.computer.suite.unit;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.config.TypedOption;
+import org.apache.hugegraph.driver.HugeClient;
+import org.apache.hugegraph.testutil.Assert;
+import org.apache.hugegraph.testutil.Whitebox;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
+import org.apache.logging.log4j.LogManager;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+
 import com.baidu.hugegraph.computer.core.common.ComputerContext;
 import com.baidu.hugegraph.computer.core.common.Constants;
 import com.baidu.hugegraph.computer.core.common.exception.ComputerException;
@@ -44,19 +60,6 @@
 import com.baidu.hugegraph.computer.core.store.entry.EntryInputImpl;
 import com.baidu.hugegraph.computer.core.util.ComputerContextUtil;
 import com.baidu.hugegraph.computer.core.worker.MockComputationParams;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.config.TypedOption;
-import org.apache.hugegraph.driver.HugeClient;
-import org.apache.hugegraph.testutil.Assert;
-import org.apache.hugegraph.testutil.Whitebox;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
 
 public class UnitTestBase {
 
@@ -76,6 +79,8 @@ protected static void clearAll() {
 
     @BeforeClass
     public static void step() throws ClassNotFoundException {
+        Runtime.getRuntime().addShutdownHook(new Thread(LogManager::shutdown));
+
         LOG.info("Setup for UnitTestSuite of hugegraph-computer");
 
         Whitebox.setInternalState(ComputerOptions.BSP_ETCD_ENDPOINTS,