Skip to content

Commit

Permalink
feat(exit): add exit manager
Browse files Browse the repository at this point in the history
  • Loading branch information
halibobo1205 committed Dec 4, 2024
1 parent 56a467c commit 90b7f96
Show file tree
Hide file tree
Showing 22 changed files with 251 additions and 136 deletions.
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ subprojects {
}
tasks.withType(Test).configureEach {
// https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:environment
/* environment 'CI', 'true'*/
environment 'CI', 'true'
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@
import org.iq80.leveldb.WriteBatch;
import org.iq80.leveldb.WriteOptions;
import org.slf4j.LoggerFactory;
import org.tron.common.exit.ExitManager;
import org.tron.common.exit.ExitReason;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.storage.metric.DbStat;
Expand Down Expand Up @@ -176,13 +178,14 @@ private void openDatabase(Options dbOptions) throws IOException {
dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles());
}
} catch (IOException e) {
String info;
if (e.getMessage().contains("Corruption:")) {
logger.error("Database {} corrupted, please delete database directory({}) and restart.",
dataBaseName, parentPath, e);
info = String.format("Database %s corrupted, please delete database directory(%s) "
+ "and restart.", dataBaseName, parentPath);
} else {
logger.error("Open Database {} failed", dataBaseName, e);
info = String.format("Open Database %s failed", dataBaseName);
}
System.exit(1);
ExitManager.getInstance().exit(ExitReason.DATABASE_ERROR, info, e);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions;
import org.slf4j.LoggerFactory;
import org.tron.common.exit.ExitManager;
import org.tron.common.exit.ExitReason;
import org.tron.common.setting.RocksDbSettings;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.storage.metric.DbStat;
Expand Down Expand Up @@ -239,13 +241,14 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) {
try {
database = RocksDB.open(options, dbPath.toString());
} catch (RocksDBException e) {
String info;
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
logger.error("Database {} corrupted, please delete database directory({}) "
+ "and restart.", dataBaseName, parentPath, e);
info = String.format("Database %s corrupted, please delete database directory(%s) "
+ "and restart.", dataBaseName, parentPath);
} else {
logger.error("Open Database {} failed", dataBaseName, e);
info = String.format("Open Database %s failed", dataBaseName);
}
System.exit(1);
ExitManager.getInstance().exit(ExitReason.DATABASE_ERROR, info, e);
}

alive = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.tron.common.error.TronDBException;
import org.tron.common.es.ExecutorServiceManager;
import org.tron.common.exit.ExitManager;
import org.tron.common.exit.ExitReason;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.utils.FileUtil;
Expand All @@ -41,6 +43,7 @@
import org.tron.core.db2.common.Key;
import org.tron.core.db2.common.Value;
import org.tron.core.db2.common.WrappedByteArray;
import org.tron.core.exception.ConfigException;
import org.tron.core.exception.RevokingStoreIllegalStateException;
import org.tron.core.store.CheckPointV2Store;
import org.tron.core.store.CheckTmpStore;
Expand Down Expand Up @@ -68,7 +71,6 @@ public class SnapshotManager implements RevokingDatabase {

private volatile int flushCount = 0;

private Thread exitThread;
private volatile boolean hitDown;

private Map<String, ListeningExecutorService> flushServices = new HashMap<>();
Expand Down Expand Up @@ -105,15 +107,6 @@ public void init() {
}
}, 10000, 3600, TimeUnit.MILLISECONDS);
}
exitThread = new Thread(() -> {
LockSupport.park();
// to Guarantee Some other thread invokes unpark with the current thread as the target
if (hitDown) {
System.exit(1);
}
});
exitThread.setName("exit-thread");
exitThread.start();
}

public static String simpleDecode(byte[] bytes) {
Expand Down Expand Up @@ -281,13 +274,6 @@ public void shutdown() {
ExecutorServiceManager.shutdownAndAwaitTermination(pruneCheckpointThread, pruneName);
flushServices.forEach((key, value) -> ExecutorServiceManager.shutdownAndAwaitTermination(value,
"flush-service-" + key));
try {
exitThread.interrupt();
// help GC
exitThread = null;
} catch (Exception e) {
logger.warn("exitThread interrupt error", e);
}
}

public void updateSolidity(int hops) {
Expand Down Expand Up @@ -365,9 +351,9 @@ public void flush() {
System.currentTimeMillis() - checkPointEnd
);
} catch (TronDBException e) {
logger.error(" Find fatal error, program will be exited soon.", e);
hitDown = true;
LockSupport.unpark(exitThread);
ExitManager.getInstance().exit(ExitReason.DATABASE_ERROR,
"Find fatal error, program will be exited soon.", e);
}
}
}
Expand Down Expand Up @@ -490,10 +476,10 @@ public void check() {
if (!isV2Open()) {
List<String> cpList = getCheckpointList();
if (cpList != null && cpList.size() != 0) {
logger.error("checkpoint check failed, the checkpoint version of database not match your " +
"config file, please set storage.checkpoint.version = 2 in your config file " +
"and restart the node.");
System.exit(-1);
String info = "checkpoint check failed, the checkpoint version of database not match your "
+ "config file, please set storage.checkpoint.version = 2 in your config file "
+ "and restart the node.";
ExitManager.getInstance().exit(ExitReason.CONFIG_ERROR, new ConfigException(info));
}
checkV1();
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.springframework.stereotype.Component;
import org.tron.common.error.TronDBException;
import org.tron.common.es.ExecutorServiceManager;
import org.tron.common.exit.ExitManager;
import org.tron.common.exit.ExitReason;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.utils.ByteArray;
import org.tron.common.utils.MerkleRoot;
Expand Down Expand Up @@ -120,8 +122,8 @@ private void maybeRun() {
}
}
} catch (Exception e) {
logger.error(" Find fatal error, program will be exited soon.", e);
System.exit(1);
ExitManager.getInstance().exit(ExitReason.DATABASE_ERROR,
"Find fatal error, program will be exited soon", e);
}
}

Expand Down
77 changes: 77 additions & 0 deletions common/src/main/java/org/tron/common/exit/ExitManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package org.tron.common.exit;

import java.util.Arrays;
import java.util.concurrent.ThreadFactory;

import com.google.common.base.Strings;
import lombok.extern.slf4j.Slf4j;

@Slf4j(topic = "Exit")
public class ExitManager {

private static final ExitManager INSTANCE = new ExitManager();
private static final String[] CI_ENVIRONMENT_VARIABLES = {
"CI",
"JENKINS_URL",
"TRAVIS",
"CIRCLECI",
"GITHUB_ACTIONS",
"GITLAB_CI"
};

private final ThreadFactory exitThreadFactory = r -> {
Thread thread = new Thread(r, "System-Exit-Thread");
thread.setDaemon(true);
return thread;
};

private ExitManager() {
}

public static ExitManager getInstance() {
return INSTANCE;
}

public void exit(ExitReason reason) {
exit(reason, reason.getDescription());
}

public void exit(ExitReason reason, String message) {
exit(reason, message, null);
}

public void exit(ExitReason reason, Throwable cause) {
exit(reason, cause.getMessage(), cause);
}

public void exit(ExitReason reason, String message, Throwable cause) {
TronExitException exitException = new TronExitException(reason, message, cause);
if (isRunningInCI()) {
if (exitException.getExitReason() != ExitReason.NORMAL_SHUTDOWN) {
throw exitException;
}
} else {
logAndExit(exitException);
}
}

private boolean isRunningInCI() {
return Arrays.stream(CI_ENVIRONMENT_VARIABLES).anyMatch(System.getenv()::containsKey);
}

private void logAndExit(TronExitException exception) {
ExitReason reason = exception.getExitReason();
String message = exception.getMessage();
if (reason == ExitReason.NORMAL_SHUTDOWN) {
if (!Strings.isNullOrEmpty(message)) {
logger.info("Exiting with code: {}, {}, {}.",
reason.getCode(), reason.getDescription(), message);
}
} else {
logger.error("Exiting with code: {}, {}, {}.", reason.getCode(), reason.getDescription(),
message, exception);
}
Thread exitThread = exitThreadFactory.newThread(() -> System.exit(reason.getCode()));
exitThread.start();
}
}
22 changes: 22 additions & 0 deletions common/src/main/java/org/tron/common/exit/ExitReason.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package org.tron.common.exit;

import lombok.Getter;

@Getter
public enum ExitReason {
NORMAL_SHUTDOWN(0, "Normal"),
CONFIG_ERROR(1, "Configuration"),
INITIALIZATION_ERROR(2, "Initialization"),
DATABASE_ERROR(3, "Database"),
EVENT_ERROR(4, "Event Trigger"),
FATAL_ERROR(-1, "Fatal");


private final int code;
private final String description;

ExitReason(int code, String description) {
this.code = code;
this.description = description;
}
}
21 changes: 21 additions & 0 deletions common/src/main/java/org/tron/common/exit/TronExitException.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package org.tron.common.exit;

import lombok.Getter;


@Getter
public class TronExitException extends RuntimeException {

private final ExitReason exitReason;

public TronExitException(ExitReason exitReason, String message) {
super(message);
this.exitReason = exitReason;
}

public TronExitException(ExitReason exitReason, String message, Throwable cause) {
super(message, cause);
this.exitReason = exitReason;
}

}
16 changes: 16 additions & 0 deletions common/src/main/java/org/tron/core/exception/ConfigException.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.tron.core.exception;

public class ConfigException extends TronRuntimeException {

public ConfigException() {
super();
}

public ConfigException(String message) {
super(message);
}

public ConfigException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.tron.core.exception;

public class EventTriggerException extends TronRuntimeException {

public EventTriggerException() {
super();
}

public EventTriggerException(String message) {
super(message);
}

public EventTriggerException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.tron.core.exception;

public class GenesisBlockException extends TronRuntimeException {

public GenesisBlockException() {
super();
}

public GenesisBlockException(String message) {
super(message);
}

public GenesisBlockException(String message, Throwable cause) {
super(message, cause);
}
}
5 changes: 0 additions & 5 deletions framework/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,6 @@ dependencies {
implementation fileTree(dir: 'libs', include: '*.jar')
// end local libraries
testImplementation group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1'
// https://github.com/stefanbirkner/system-rules/issues/85
// https://openjdk.org/jeps/411, JEP 411: Deprecate the Security Manager for Removal from JDK17
// WARNING: A terminally deprecated method in java.lang.System has been called
// WARNING: System::setSecurityManager will be removed in a future release
testImplementation group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.16.0'

implementation group: 'com.google.inject', name: 'guice', version: '4.1.0'
implementation group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.2'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.tron.common.exit.ExitManager;
import org.tron.common.exit.ExitReason;
import org.tron.common.parameter.CommonParameter;
import org.tron.core.ChainBaseManager;
import org.tron.core.config.args.Args;
Expand Down Expand Up @@ -82,8 +84,8 @@ public void startServices() {
try {
services.start();
} catch (Exception e) {
logger.error("Failed to start services", e);
System.exit(1);
ExitManager.getInstance().exit(ExitReason.INITIALIZATION_ERROR,
"Failed to start services", e);
}
}

Expand Down
Loading

0 comments on commit 90b7f96

Please sign in to comment.