feat(all): add exit manager for System.exit

build.gradle

@@ -67,6 +67,10 @@ subprojects {
         reproducibleFileOrder = true
         duplicatesStrategy = DuplicatesStrategy.INCLUDE // allow duplicates
     }
+    tasks.withType(Test).configureEach {
+        // https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:environment
+        environment 'CI', 'true'
+    }
 }
 
 task copyToParent(type: Copy) {

chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java

@@ -18,7 +18,7 @@
 import static org.fusesource.leveldbjni.JniDBFactory.factory;
 
 import com.google.common.collect.Sets;
-import java.io.File;
+import com.google.common.primitives.Bytes;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -30,18 +30,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
-
-import com.google.common.primitives.Bytes;
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.iq80.leveldb.CompressionType;
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.DBIterator;
 import org.iq80.leveldb.Logger;
@@ -50,6 +46,7 @@
 import org.iq80.leveldb.WriteBatch;
 import org.iq80.leveldb.WriteOptions;
 import org.slf4j.LoggerFactory;
+import org.tron.common.exit.ExitManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.storage.WriteOptionsWrapper;
 import org.tron.common.storage.metric.DbStat;
@@ -59,6 +56,7 @@
 import org.tron.core.db.common.iterator.StoreIterator;
 import org.tron.core.db2.common.Instance;
 import org.tron.core.db2.common.WrappedByteArray;
+import org.tron.core.exception.DatabaseExitException;
 
 @Slf4j(topic = "DB")
 @NoArgsConstructor
@@ -151,13 +149,14 @@ private void openDatabase(Options dbOptions) throws IOException {
                 dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles());
       }
     } catch (IOException e) {
+      String msg;
       if (e.getMessage().contains("Corruption:")) {
-        logger.error("Database {} corrupted, please delete database directory({}) and restart.",
-            dataBaseName, parentPath, e);
+        msg = String.format("Database %s corrupted, please delete database directory(%s) "
+            + "and restart.", dataBaseName, parentPath);
       } else {
-        logger.error("Open Database {} failed", dataBaseName, e);
+        msg = String.format("Open Database %s failed", dataBaseName);
       }
-      System.exit(1);
+      ExitManager.exit(msg, new DatabaseExitException(e));
     }
   }
 

chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java

@@ -36,6 +36,7 @@
 import org.rocksdb.WriteBatch;
 import org.rocksdb.WriteOptions;
 import org.slf4j.LoggerFactory;
+import org.tron.common.exit.ExitManager;
 import org.tron.common.setting.RocksDbSettings;
 import org.tron.common.storage.WriteOptionsWrapper;
 import org.tron.common.storage.metric.DbStat;
@@ -45,6 +46,7 @@
 import org.tron.core.db.common.iterator.RockStoreIterator;
 import org.tron.core.db2.common.Instance;
 import org.tron.core.db2.common.WrappedByteArray;
+import org.tron.core.exception.DatabaseExitException;
 
 
 @Slf4j(topic = "DB")
@@ -266,13 +268,14 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) {
           try {
             database = RocksDB.open(options, dbPath.toString());
           } catch (RocksDBException e) {
+            String msg;
             if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
-              logger.error("Database {} corrupted, please delete database directory({}) " +
-                      "and restart.", dataBaseName, parentPath, e);
+              msg = String.format("Database %s corrupted, please delete database directory(%s) "
+                  + "and restart.", dataBaseName, parentPath);
             } else {
-              logger.error("Open Database {} failed", dataBaseName, e);
+              msg = String.format("Open Database %s failed", dataBaseName);
             }
-            System.exit(1);
+            ExitManager.exit(msg, new DatabaseExitException(e));
           }
 
           alive = true;

chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java

@@ -20,7 +20,6 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.LockSupport;
 import java.util.stream.Collectors;
 import javax.annotation.PostConstruct;
 import lombok.Getter;
@@ -29,6 +28,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.tron.common.error.TronDBException;
 import org.tron.common.es.ExecutorServiceManager;
+import org.tron.common.exit.ExitManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.storage.WriteOptionsWrapper;
 import org.tron.common.utils.FileUtil;
@@ -41,6 +41,8 @@
 import org.tron.core.db2.common.Key;
 import org.tron.core.db2.common.Value;
 import org.tron.core.db2.common.WrappedByteArray;
+import org.tron.core.exception.ConfigExitException;
+import org.tron.core.exception.DatabaseExitException;
 import org.tron.core.exception.RevokingStoreIllegalStateException;
 import org.tron.core.store.CheckPointV2Store;
 import org.tron.core.store.CheckTmpStore;
@@ -68,7 +70,6 @@ public class SnapshotManager implements RevokingDatabase {
 
   private volatile int flushCount = 0;
 
-  private Thread exitThread;
   private volatile boolean  hitDown;
 
   private Map<String, ListeningExecutorService> flushServices = new HashMap<>();
@@ -105,15 +106,6 @@ public void init() {
         }
       }, 10000, 3600, TimeUnit.MILLISECONDS);
     }
-    exitThread =  new Thread(() -> {
-      LockSupport.park();
-      // to Guarantee Some other thread invokes unpark with the current thread as the target
-      if (hitDown) {
-        System.exit(1);
-      }
-    });
-    exitThread.setName("exit-thread");
-    exitThread.start();
   }
 
   public static String simpleDecode(byte[] bytes) {
@@ -281,13 +273,6 @@ public void shutdown() {
     ExecutorServiceManager.shutdownAndAwaitTermination(pruneCheckpointThread, pruneName);
     flushServices.forEach((key, value) -> ExecutorServiceManager.shutdownAndAwaitTermination(value,
         "flush-service-" + key));
-    try {
-      exitThread.interrupt();
-      // help GC
-      exitThread = null;
-    } catch (Exception e) {
-      logger.warn("exitThread interrupt error", e);
-    }
   }
 
   public void updateSolidity(int hops) {
@@ -365,9 +350,9 @@ public void flush() {
             System.currentTimeMillis() - checkPointEnd
         );
       } catch (TronDBException e) {
-        logger.error(" Find fatal error, program will be exited soon.", e);
+        String msg = " Find fatal error, program will be exited soon.";
         hitDown = true;
-        LockSupport.unpark(exitThread);
+        ExitManager.exit(msg, new DatabaseExitException(e));
       }
     }
   }
@@ -490,10 +475,10 @@ public void check() {
     if (!isV2Open()) {
       List<String> cpList = getCheckpointList();
       if (cpList != null && cpList.size() != 0) {
-        logger.error("checkpoint check failed, the checkpoint version of database not match your " +
-            "config file, please set storage.checkpoint.version = 2 in your config file " +
-            "and restart the node.");
-        System.exit(-1);
+        String msg = "checkpoint check failed, the checkpoint version of database not match your "
+            + "config file, please set storage.checkpoint.version = 2 in your config file "
+            + "and restart the node.";
+        ExitManager.exit(new ConfigExitException(msg));
       }
       checkV1();
     } else {

chainbase/src/main/java/org/tron/core/service/RewardViCalService.java

@@ -23,13 +23,15 @@
 import org.springframework.stereotype.Component;
 import org.tron.common.error.TronDBException;
 import org.tron.common.es.ExecutorServiceManager;
+import org.tron.common.exit.ExitManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.utils.ByteArray;
 import org.tron.common.utils.MerkleRoot;
 import org.tron.common.utils.Pair;
 import org.tron.common.utils.Sha256Hash;
 import org.tron.core.db.common.iterator.DBIterator;
 import org.tron.core.db2.common.DB;
+import org.tron.core.exception.DatabaseExitException;
 import org.tron.core.store.DelegationStore;
 import org.tron.core.store.DynamicPropertiesStore;
 import org.tron.core.store.RewardViStore;
@@ -120,8 +122,8 @@ private void maybeRun() {
         }
       }
     } catch (Exception e) {
-      logger.error(" Find fatal error, program will be exited soon.", e);
-      System.exit(1);
+      String msg = " Find fatal error, program will be exited soon.";
+      ExitManager.exit(msg, new DatabaseExitException(e));
     }
   }
 

common/src/main/java/org/tron/common/exit/ExitManager.java

@@ -0,0 +1,79 @@
+package org.tron.common.exit;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.ThreadFactory;
+import lombok.extern.slf4j.Slf4j;
+import org.tron.core.exception.TronExitException;
+
+@Slf4j(topic = "Exit")
+public class ExitManager {
+
+  private static final String[] CI_ENVIRONMENT_VARIABLES = {
+      "CI",
+      "JENKINS_URL",
+      "TRAVIS",
+      "CIRCLECI",
+      "GITHUB_ACTIONS",
+      "GITLAB_CI"
+  };
+
+  private static final int EXIT_CODE_NORMAL = 0;
+
+  private static final ThreadFactory exitThreadFactory = r -> {
+    Thread thread = new Thread(r, "System-Exit-Thread");
+    thread.setDaemon(true);
+    return thread;
+  };
+
+  private ExitManager() {
+  }
+
+  public static void exit() {
+    exit((String) null);
+  }
+
+  public static void exit(String msg) {
+    exit(msg, null);
+  }
+
+  public static void exit(TronExitException cause) {
+    exit(cause.getMessage(), cause);
+  }
+
+  public static void exit(String msg, TronExitException cause) {
+    TronExit exit = new TronExit(msg, cause);
+    if (isRunningInCI()) {
+      if (Objects.nonNull(cause)) {
+        throw cause;
+      } else if (Objects.nonNull(msg)) {
+        logger.info("{}", msg);
+      }
+    } else {
+      logAndExit(exit);
+    }
+  }
+
+  private static boolean isRunningInCI() {
+    return Arrays.stream(CI_ENVIRONMENT_VARIABLES).anyMatch(System.getenv()::containsKey);
+  }
+
+  private static void logAndExit(TronExit exit) {
+    String msg = exit.getMsg();
+    TronExitException cause = exit.getException();
+    final int code = Objects.isNull(cause) ? EXIT_CODE_NORMAL : cause.getExitCode();
+    if (code == EXIT_CODE_NORMAL) {
+      if (Objects.nonNull(msg)) {
+        logger.info("Exiting, {}.", msg);
+      }
+    } else {
+      if (Objects.isNull(msg)) {
+        logger.error("Exiting with code: {}.", code, cause);
+      } else {
+        logger.error("Exiting with code: {}, {}.", code, msg, cause);
+      }
+    }
+    Thread exitThread = exitThreadFactory.newThread(() -> System.exit(code));
+    exitThread.start();
+  }
+}

common/src/main/java/org/tron/common/exit/TronExit.java

@@ -0,0 +1,13 @@
+package org.tron.common.exit;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.tron.core.exception.TronExitException;
+
+@Getter
+@AllArgsConstructor
+public class TronExit {
+
+  private String msg;
+  private TronExitException exception;
+}

common/src/main/java/org/tron/core/exception/ConfigExitException.java

@@ -0,0 +1,19 @@
+package org.tron.core.exception;
+
+public class ConfigExitException extends TronExitException {
+
+  public ConfigExitException(String message) {
+    super(message);
+    setExitCode(1);
+  }
+
+  public ConfigExitException(String message, Throwable cause) {
+    super(message, cause);
+    setExitCode(1);
+  }
+
+  public ConfigExitException(Throwable cause) {
+    super(cause);
+    setExitCode(1);
+  }
+}

common/src/main/java/org/tron/core/exception/DatabaseExitException.java

@@ -0,0 +1,19 @@
+package org.tron.core.exception;
+
+public class DatabaseExitException extends TronExitException {
+
+  public DatabaseExitException(String message) {
+    super(message);
+    setExitCode(2);
+  }
+
+  public DatabaseExitException(String message, Throwable cause) {
+    super(message, cause);
+    setExitCode(2);
+  }
+
+  public DatabaseExitException(Throwable cause) {
+    super(cause);
+    setExitCode(2);
+  }
+}

common/src/main/java/org/tron/core/exception/EventExitException.java

@@ -0,0 +1,19 @@
+package org.tron.core.exception;
+
+public class EventExitException extends TronExitException {
+
+  public EventExitException(String message) {
+    super(message);
+    setExitCode(3);
+  }
+
+  public EventExitException(String message, Throwable cause) {
+    super(message, cause);
+    setExitCode(3);
+  }
+
+  public EventExitException(Throwable cause) {
+    super(cause);
+    setExitCode(3);
+  }
+}