feat(dist): support auto reload config & compress audit log (#1492)

Also enhance: - add time strategy to avoid old logs overwriting by current logs - auto create log folder if it does not exist - add some comment for auth config * use strict mode for config property & separate rest configs * reduce log format length * Update rest-server.properties * update all configs (TODO: push to other modules)
apache · Jun 18, 2021 · fceaaa2 · fceaaa2
1 parent 0935c75
commit fceaaa2
Show file tree

Hide file tree

Showing 7 changed files with 214 additions and 53 deletions.
diff --git a/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh
@@ -38,31 +38,35 @@ function abs_path() {
     echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 }
 
-BIN=`abs_path`
-TOP="$(cd $BIN/../ && pwd)"
+BIN=$(abs_path)
+TOP="$(cd "$BIN"/../ && pwd)"
 CONF="$TOP/conf"
 LOGS="$TOP/logs"
 PID_FILE="$BIN/pid"
 
-. $BIN/util.sh
+. "$BIN"/util.sh
 
-GREMLIN_SERVER_URL=`read_property "$CONF/rest-server.properties" "gremlinserver.url"`
+GREMLIN_SERVER_URL=$(read_property "$CONF/rest-server.properties" "gremlinserver.url")
 if [ -z "$GREMLIN_SERVER_URL" ]; then
     GREMLIN_SERVER_URL="http://127.0.0.1:8182"
 fi
-REST_SERVER_URL=`read_property "$CONF/rest-server.properties" "restserver.url"`
+REST_SERVER_URL=$(read_property "$CONF/rest-server.properties" "restserver.url")
 
 check_port "$GREMLIN_SERVER_URL"
 check_port "$REST_SERVER_URL"
 
+if [ ! -d "$LOGS" ]; then
+    mkdir -p "$LOGS"
+fi
+
 echo "Starting HugeGraphServer..."
 
 "$BIN"/hugegraph-server.sh "$CONF"/gremlin-server.yaml "$CONF"/rest-server.properties \
 "$OPEN_SECURITY_CHECK" "$USER_OPTION" "$GC_OPTION" >>"$LOGS/hugegraph-server.log" 2>&1 &
 
 PID="$!"
 # Write pid to file
-echo "$PID" > $PID_FILE
+echo "$PID" > "$PID_FILE"
 
 trap 'kill $PID; exit' SIGHUP SIGINT SIGQUIT SIGTERM
 
@@ -73,7 +77,7 @@ wait_for_startup ${PID} 'HugeGraphServer' "$REST_SERVER_URL/graphs" ${SERVER_STA
 disown
 
 if [ "$OPEN_MONITOR" == "true" ]; then
-    $BIN/start-monitor.sh
+    "$BIN"/start-monitor.sh
     if [ $? -ne 0 ]; then
         echo "Failed to open monitor, please start it manually"
     fi

diff --git a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties
@@ -1,4 +1,5 @@
-# gremlin entrence to create graph
+# gremlin entrance to create graph
+# auth config: com.baidu.hugegraph.auth.HugeFactoryAuthProxy
 gremlin.graph=com.baidu.hugegraph.HugeFactory
 
 # cache config

diff --git a/hugegraph-dist/src/assembly/static/conf/log4j2.xml b/hugegraph-dist/src/assembly/static/conf/log4j2.xml
@@ -1,29 +1,65 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<configuration status="error">
+<!-- Config will be auto loaded every 60s -->
+<configuration status="error" monitorInterval="60">
+    <properties>
+        <property name="LOG_PATH">logs</property>
+        <property name="FILE_NAME">hugegraph-server</property>
+    </properties>
+
     <appenders>
         <Console name="console" target="SYSTEM_OUT">
             <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
         </Console>
 
         <!-- Normal server log config -->
-        <RollingRandomAccessFile name="file" fileName="logs/hugegraph-server.log"
-            filePattern="logs/$${date:yyyy-MM}/hugegraph-server-%d{yyyy-MM-dd}-%i.log"
+        <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
             immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+            <!-- Trigger after exceeding 1day or 50MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 5 files per day & auto delete after over 2GB or 100 files -->
+            <DefaultRolloverStrategy max="5">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.log"/>
+                    <!-- Limit log amount & size -->
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="2GB" />
+                        <IfAccumulatedFileCount exceeds="100" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
 
-        <!-- Separate audit log, buffer size is 512KB -->
-        <RollingRandomAccessFile name="audit" fileName="logs/audit-hugegraph-server.log"
-            filePattern="logs/$${date:yyyy-MM}/audit-hugegraph-server-%d{yyyy-MM-dd}-%i.log"
+        <!-- Separate & compress audit log, buffer size is 512KB -->
+        <RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
             bufferSize="524288" immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
             <!-- Use simple format for audit log to speed up -->
             <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <!-- Trigger after exceeding 1hour or 500MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="500MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 2 files per hour & auto delete [after 60 days] or [over 5GB or 500 files] -->
+            <DefaultRolloverStrategy max="2">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.gz"/>
+                    <IfLastModified age="60d"/>
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="5GB" />
+                        <IfAccumulatedFileCount exceeds="500" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
     </appenders>
 

diff --git a/hugegraph-dist/src/assembly/static/conf/rest-server.properties b/hugegraph-dist/src/assembly/static/conf/rest-server.properties
@@ -10,21 +10,33 @@ graphs=[hugegraph:conf/hugegraph.properties]
 batch.max_write_ratio=80
 batch.max_write_threads=0
 
-# authentication
+# authentication configs
+# choose 'com.baidu.hugegraph.auth.StandardAuthenticator' or 'com.baidu.hugegraph.auth.ConfigAuthenticator'
 #auth.authenticator=
-#auth.admin_token=
-#auth.user_tokens=[]
+
+# for StandardAuthenticator mode
+#auth.graph_store=hugegraph
+# auth client config
 #auth.remote_url=127.0.0.1:8899,127.0.0.1:8898,127.0.0.1:8897
 
-server.id=server-1
-server.role=master
+# for ConfigAuthenticator mode
+#auth.admin_token=
+#auth.user_tokens=[]
 
+# rpc group configs of multi graph servers
+# rpc server configs
 rpc.server_host=127.0.0.1
 rpc.server_port=8090
 #rpc.server_timeout=30
+
+# rpc client configs (like enable to keep cache consistency)
 rpc.remote_url=127.0.0.1:8090
 #rpc.client_connect_timeout=20
 #rpc.client_reconnect_period=10
 #rpc.client_read_timeout=40
 #rpc.client_retries=3
 #rpc.client_load_balancer=consistentHash
+
+# lightweight load balancing (beta)
+server.id=server-1
+server.role=master
diff --git a/hugegraph-dist/src/main/resources/log4j2.xml b/hugegraph-dist/src/main/resources/log4j2.xml
@@ -1,29 +1,65 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<configuration status="error">
+<!-- Config will be auto loaded every 60s -->
+<configuration status="error" monitorInterval="60">
+    <properties>
+        <property name="LOG_PATH">logs</property>
+        <property name="FILE_NAME">hugegraph-server</property>
+    </properties>
+
     <appenders>
         <Console name="console" target="SYSTEM_OUT">
             <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
         </Console>
 
         <!-- Normal server log config -->
-        <RollingRandomAccessFile name="file" fileName="logs/hugegraph-server.log"
-            filePattern="logs/$${date:yyyy-MM}/hugegraph-server-%d{yyyy-MM-dd}-%i.log"
+        <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
             immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+            <!-- Trigger after exceeding 1day or 50MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 5 files per day & auto delete after over 2GB or 100 files -->
+            <DefaultRolloverStrategy max="5">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.log"/>
+                    <!-- Limit log amount & size -->
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="2GB" />
+                        <IfAccumulatedFileCount exceeds="100" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
 
-        <!-- Separate audit log, buffer size is 512KB -->
-        <RollingRandomAccessFile name="audit" fileName="logs/audit-hugegraph-server.log"
-            filePattern="logs/$${date:yyyy-MM}/audit-hugegraph-server-%d{yyyy-MM-dd}-%i.log"
+        <!-- Separate & compress audit log, buffer size is 512KB -->
+        <RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
             bufferSize="524288" immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
             <!-- Use simple format for audit log to speed up -->
             <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <!-- Trigger after exceeding 1hour or 500MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="500MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 2 files per hour & auto delete [after 60 days] or [over 5GB or 500 files] -->
+            <DefaultRolloverStrategy max="2">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.gz"/>
+                    <IfLastModified age="60d"/>
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="5GB" />
+                        <IfAccumulatedFileCount exceeds="500" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
     </appenders>
 

diff --git a/hugegraph-example/src/main/resources/log4j2.xml b/hugegraph-example/src/main/resources/log4j2.xml
@@ -1,29 +1,65 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<configuration status="error">
+<!-- Config will be auto loaded every 60s -->
+<configuration status="error" monitorInterval="60">
+    <properties>
+        <property name="LOG_PATH">logs</property>
+        <property name="FILE_NAME">hugegraph-example</property>
+    </properties>
+
     <appenders>
         <Console name="console" target="SYSTEM_OUT">
             <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
         </Console>
 
         <!-- Normal server log config -->
-        <RollingRandomAccessFile name="file" fileName="logs/hugegraph-example.log"
-            filePattern="logs/$${date:yyyy-MM}/hugegraph-example-%d{yyyy-MM-dd}-%i.log"
+        <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
             immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
-            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} %-5r [%t] [%-5p] %c %x - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+            <!-- Trigger after exceeding 1day or 50MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 5 files per day & auto delete after over 2GB or 100 files -->
+            <DefaultRolloverStrategy max="5">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.log"/>
+                    <!-- Limit log amount & size -->
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="2GB" />
+                        <IfAccumulatedFileCount exceeds="100" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
 
-        <!-- Separate audit log, buffer size is 512KB -->
-        <RollingRandomAccessFile name="audit" fileName="logs/audit-hugegraph-example.log"
-            filePattern="logs/$${date:yyyy-MM}/audit-hugegraph-example-%d{yyyy-MM-dd}-%i.log"
+        <!-- Separate & compress audit log, buffer size is 512KB -->
+        <RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
+            filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
             bufferSize="524288" immediateFlush="false">
             <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
             <!-- Use simple format for audit log to speed up -->
             <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/>
-            <SizeBasedTriggeringPolicy size="50MB"/>
+            <!-- Trigger after exceeding 1hour or 500MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="500MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 2 files per hour & auto delete [after 60 days] or [over 5GB or 500 files] -->
+            <DefaultRolloverStrategy max="2">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.gz"/>
+                    <IfLastModified age="60d"/>
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="5GB" />
+                        <IfAccumulatedFileCount exceeds="500" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
         </RollingRandomAccessFile>
     </appenders>