Merge branch 'mempool' into junderw/perf-precache-speed

mempool · Oct 3, 2023 · 69c97c1 · 69c97c1
2 parents 3e28319 + e663693
commit 69c97c1
Show file tree

Hide file tree

Showing 14 changed files with 481 additions and 99 deletions.
diff --git a/electrs-start-liquid b/electrs-start-liquid
@@ -20,6 +20,8 @@ do
 		-- \
 		--network liquid \
 		--http-socket-file "${HOME}/socket/esplora-liquid-mainnet" \
+		--rpc-socket-file "${HOME}/socket/electrum-liquid-mainnet" \
+		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" \
 		--asset-db-path "${HOME}/asset_registry_db" \
 		--daemon-dir "${HOME}" \
 		--db-dir /electrs \

diff --git a/electrs-start-liquidtestnet b/electrs-start-liquidtestnet
@@ -20,6 +20,8 @@ do
 		-- \
 		--network liquidtestnet \
 		--http-socket-file "${HOME}/socket/esplora-liquid-testnet" \
+		--rpc-socket-file "${HOME}/socket/electrum-liquid-testnet" \
+		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" \
 		--asset-db-path "${HOME}/asset_registry_testnet_db" \
 		--daemon-dir "${HOME}" \
 		--db-dir "/electrs" \

diff --git a/electrs-start-mainnet b/electrs-start-mainnet
@@ -3,11 +3,23 @@
 #export PATH="${HOME}/.cargo/bin:${PATH}"
 
 # don't bother making electrs.core files
-ulimit -c
+ulimit -c 0
 
 # get credentials from bitcoin.conf directly
-BITCOIN_RPC_USER=$(grep 'rpcuser=' ${HOME}/bitcoin.conf|cut -d = -f2|head -1)
-BITCOIN_RPC_PASS=$(grep 'rpcpassword=' ${HOME}/bitcoin.conf|cut -d = -f2|head -1)
+BITCOIN_RPC_USER=$(grep 'rpcuser=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
+BITCOIN_RPC_PASS=$(grep 'rpcpassword=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
+
+# base limits
+UTXOS_LIMIT=500
+ELECTRUM_TXS_LIMIT=500
+
+# increase limits based on hostname
+NODENAME=$(hostname|cut -d . -f1)
+LOCATION=$(hostname|cut -d . -f2)
+if [ "${NODENAME}" = "node201" ];then
+	UTXOS_LIMIT=9000
+	ELECTRUM_TXS_LIMIT=9000
+fi
 
 # run in loop in case of crash
 until false
@@ -42,11 +54,14 @@ do
 		--http-socket-file "${HOME}/socket/esplora-bitcoin-mainnet" \
 		--precache-scripts "${PS_FOLDER}/popular-scripts.txt" \
 		--precache-threads 100 \
+		--rpc-socket-file "${HOME}/socket/electrum-bitcoin-mainnet" \
 		--daemon-dir "${HOME}" \
 		--db-dir "/electrs" \
 		--cookie "${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASS}" \
 		--cors '*' \
 		--address-search \
+		--utxos-limit "${UTXOS_LIMIT}" \
+		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
 		-vvv
 	sleep 1
 done
diff --git a/electrs-start-signet b/electrs-start-signet
@@ -3,12 +3,24 @@
 #export PATH="${HOME}/.cargo/bin:${PATH}"
 
 # don't bother making electrs.core files
-ulimit -c
+ulimit -c 0
 
 # get credentials from bitcoin.conf directly
 BITCOIN_RPC_USER=$(grep 'rpcuser=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
 BITCOIN_RPC_PASS=$(grep 'rpcpassword=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
 
+# base limits
+UTXOS_LIMIT=500
+ELECTRUM_TXS_LIMIT=500
+
+# increase limits based on hostname
+NODENAME=$(hostname|cut -d . -f1)
+LOCATION=$(hostname|cut -d . -f2)
+if [ "${NODENAME}" = "node201" ];then
+	UTXOS_LIMIT=9000
+	ELECTRUM_TXS_LIMIT=9000
+fi
+
 # run in loop in case of crash
 until false
 do
@@ -19,11 +31,15 @@ do
 		-- \
 		--network signet \
 		--http-socket-file "${HOME}/socket/esplora-bitcoin-signet" \
+		--rpc-socket-file "${HOME}/socket/electrum-bitcoin-signet" \
+		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" \
 		--daemon-dir "${HOME}" \
 		--db-dir "/electrs" \
 		--cookie "${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASS}" \
 		--cors '*' \
 		--address-search \
+		--utxos-limit "${UTXOS_LIMIT}" \
+		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
 		-vvv
 	sleep 1
 done
diff --git a/electrs-start-testnet b/electrs-start-testnet
@@ -3,11 +3,23 @@
 #export PATH="${HOME}/.cargo/bin:${PATH}"
 
 # don't bother making electrs.core files
-ulimit -c
+ulimit -c 0
 
 # get credentials from bitcoin.conf directly
-BITCOIN_RPC_USER=$(grep 'rpcuser=' ${HOME}/bitcoin.conf|cut -d = -f2|head -1)
-BITCOIN_RPC_PASS=$(grep 'rpcpassword=' ${HOME}/bitcoin.conf|cut -d = -f2|head -1)
+BITCOIN_RPC_USER=$(grep 'rpcuser=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
+BITCOIN_RPC_PASS=$(grep 'rpcpassword=' "${HOME}/bitcoin.conf"|cut -d = -f2|head -1)
+
+# base limits
+UTXOS_LIMIT=500
+ELECTRUM_TXS_LIMIT=500
+
+# increase limits based on hostname
+NODENAME=$(hostname|cut -d . -f1)
+LOCATION=$(hostname|cut -d . -f2)
+if [ "${NODENAME}" = "node201" ];then
+	UTXOS_LIMIT=9000
+	ELECTRUM_TXS_LIMIT=9000
+fi
 
 # run in loop in case of crash
 until false
@@ -19,11 +31,15 @@ do
 		-- \
 		--network testnet \
 		--http-socket-file "${HOME}/socket/esplora-bitcoin-testnet" \
+		--rpc-socket-file "${HOME}/socket/electrum-bitcoin-testnet" \
+		--precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" \
 		--daemon-dir "${HOME}" \
 		--db-dir "/electrs" \
 		--cookie "${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASS}" \
 		--cors '*' \
 		--address-search \
+		--utxos-limit "${UTXOS_LIMIT}" \
+		--electrum-txs-limit "${ELECTRUM_TXS_LIMIT}" \
 		-vvv
 	sleep 1
 done
diff --git a/src/bin/electrs.rs b/src/bin/electrs.rs
@@ -109,6 +109,20 @@ fn run_server(config: Arc<Config>) -> Result<()> {
     loop {
         if let Err(err) = signal.wait(Duration::from_millis(500), true) {
             info!("stopping server: {}", err);
+
+            electrs::util::spawn_thread("shutdown-thread-checker", || {
+                let mut counter = 40;
+                let interval_ms = 500;
+
+                while counter > 0 {
+                    electrs::util::with_spawned_threads(|threads| {
+                        debug!("Threads during shutdown: {:?}", threads);
+                    });
+                    std::thread::sleep(std::time::Duration::from_millis(interval_ms));
+                    counter -= 1;
+                }
+            });
+
             rest_server.stop();
             // the electrum server is stopped when dropped
             break;
@@ -137,4 +151,7 @@ fn main() {
         error!("server failed: {}", e.display_chain());
         process::exit(1);
     }
+    electrs::util::with_spawned_threads(|threads| {
+        debug!("Threads before closing: {:?}", threads);
+    });
 }
diff --git a/src/config.rs b/src/config.rs
@@ -41,6 +41,7 @@ pub struct Config {
     pub electrum_rpc_addr: SocketAddr,
     pub http_addr: SocketAddr,
     pub http_socket_file: Option<PathBuf>,
+    pub rpc_socket_file: Option<PathBuf>,
     pub monitoring_addr: SocketAddr,
     pub jsonrpc_import: bool,
     pub light_mode: bool,
@@ -250,6 +251,14 @@ impl Config {
                     .takes_value(true),
             );
 
+        #[cfg(unix)]
+        let args = args.arg(
+                Arg::with_name("rpc_socket_file")
+                    .long("rpc-socket-file")
+                    .help("Electrum RPC 'unix socket file' to listen on (default disabled, enabling this ignores the electrum_rpc_addr arg)")
+                    .takes_value(true),
+            );
+
         #[cfg(feature = "liquid")]
         let args = args
             .arg(
@@ -391,6 +400,7 @@ impl Config {
         );
 
         let http_socket_file: Option<PathBuf> = m.value_of("http_socket_file").map(PathBuf::from);
+        let rpc_socket_file: Option<PathBuf> = m.value_of("rpc_socket_file").map(PathBuf::from);
         let monitoring_addr: SocketAddr = str_to_socketaddr(
             m.value_of("monitoring_addr")
                 .unwrap_or(&format!("127.0.0.1:{}", default_monitoring_port)),
@@ -459,6 +469,7 @@ impl Config {
             electrum_banner,
             http_addr,
             http_socket_file,
+            rpc_socket_file,
             monitoring_addr,
             mempool_backlog_stats_ttl: value_t_or_exit!(m, "mempool_backlog_stats_ttl", u64),
             mempool_recent_txs_size: value_t_or_exit!(m, "mempool_recent_txs_size", usize),

diff --git a/src/daemon.rs b/src/daemon.rs
@@ -320,10 +320,13 @@ impl Daemon {
             let info = daemon.getblockchaininfo()?;
             let mempool = daemon.getmempoolinfo()?;
 
-            if mempool.loaded
-                && !info.initialblockdownload.unwrap_or(false)
-                && info.blocks == info.headers
-            {
+            let ibd_done = if network.is_regtest() {
+                info.blocks == 0 && info.headers == 0
+            } else {
+                false
+            } || !info.initialblockdownload.unwrap_or(false);
+
+            if mempool.loaded && ibd_done && info.blocks == info.headers {
                 break;
             }