Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Install popular-scripts as a cronjob #59

Merged
merged 6 commits into from
Dec 13, 2024
Merged

Feature: Install popular-scripts as a cronjob #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a8aba23
Feature: Install popular-scripts as a cronjob
junderw Oct 9, 2023
97e8708
Fix Testnet4 addition
junderw May 7, 2024
1423a4e
Merge remote-tracking branch 'mempool' into junderw/start-script-fix
junderw May 17, 2024
4a9feb9
Use nice
junderw May 17, 2024
bd1ec25
Merge branch 'junderw/fix-testnet4' into junderw/start-script-fix
junderw May 17, 2024
67ddaca
Don't crontab popular-scripts for now
wiz Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions src/chain.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ impl Network {

pub fn genesis_hash(network: Network) -> BlockHash {
#[cfg(not(feature = "liquid"))]
return bitcoin_genesis_hash(network.into());
return bitcoin_genesis_hash(network);
#[cfg(feature = "liquid")]
return liquid_genesis_hash(network);
}
Expand All @@ -139,20 +139,29 @@ pub fn bitcoin_genesis_hash(network: Network) -> bitcoin::BlockHash {
genesis_block(BNetwork::Bitcoin).block_hash();
static ref TESTNET_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Testnet).block_hash();
static ref TESTNET4_GENESIS: bitcoin::BlockHash =
BlockHash::from_str("00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043").unwrap();
static ref TESTNET4_GENESIS: bitcoin::BlockHash = bitcoin::BlockHash::from_str(
"00000000da84f2bafbbc53dee25a72ae507ff4914b867c565be350b0da8bf043"
)
.unwrap();
static ref REGTEST_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Regtest).block_hash();
static ref SIGNET_GENESIS: bitcoin::BlockHash =
genesis_block(BNetwork::Signet).block_hash();
}
#[cfg(not(feature = "liquid"))]
match network {
Network::Bitcoin => *BITCOIN_GENESIS,
Network::Testnet => *TESTNET_GENESIS,
Network::Testnet4 => *TESTNET4_GENESIS,
Network::Regtest => *REGTEST_GENESIS,
Network::Signet => *SIGNET_GENESIS,
}
#[cfg(feature = "liquid")]
match network {
Network::Liquid => *BITCOIN_GENESIS,
Network::LiquidTestnet => *TESTNET_GENESIS,
Network::LiquidRegtest => *REGTEST_GENESIS,
}
}

#[cfg(feature = "liquid")]
Expand Down
1 change: 1 addition & 0 deletions src/daemon.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ pub struct Daemon {
}

impl Daemon {
#[allow(clippy::too_many_arguments)]
pub fn new(
daemon_dir: PathBuf,
blocks_dir: PathBuf,
Expand Down
8 changes: 7 additions & 1 deletion src/elements/peg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,13 @@ pub fn get_pegout_data(
let pegged_asset_id = network.pegged_asset()?;
txout.pegout_data().filter(|pegout| {
pegout.asset == Asset::Explicit(*pegged_asset_id)
&& pegout.genesis_hash == bitcoin_genesis_hash(parent_network)
&& pegout.genesis_hash
== bitcoin_genesis_hash(match parent_network {
BNetwork::Bitcoin => Network::Liquid,
BNetwork::Testnet => Network::LiquidTestnet,
BNetwork::Signet => return false,
BNetwork::Regtest => Network::LiquidRegtest,
})
})
}

Expand Down
5 changes: 4 additions & 1 deletion src/rest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1668,7 +1668,10 @@ fn address_to_scripthash(addr: &str, network: Network) -> Result<FullHash, HttpE
// `addr_network` will be detected as Testnet for all of them.
addr_network == network
|| (addr_network == Network::Testnet
&& matches!(network, Network::Regtest | Network::Signet | Network::Testnet4))
&& matches!(
network,
Network::Regtest | Network::Signet | Network::Testnet4
))
};

#[cfg(feature = "liquid")]
Expand Down
92 changes: 67 additions & 25 deletions start
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ FEATURES=default
DB_FOLDER=/electrs
NODENAME=$(hostname|cut -d . -f1)
LOCATION=$(hostname|cut -d . -f2)
USAGE="Usage: $0 (mainnet|testnet|signet|liquid|liquidtestnet) [popular-scripts]"

# load rust if necessary
if [ -e "${HOME}/.cargo/env" ];then
Expand Down Expand Up @@ -38,34 +39,93 @@ esac
case "${1}" in
mainnet)
THREADS=$((NPROC / 3))
CRONJOB_TIMING="20 4 * * *"
;;
testnet)
NETWORK=testnet
THREADS=$((NPROC / 6))
CRONJOB_TIMING="2 4 * * *"
;;
testnet4)
NETWORK=testnet4
MAGIC=283f161c
THREADS=$((NPROC / 6))
CRONJOB_TIMING="17 4 * * *"
;;
signet)
NETWORK=signet
THREADS=$((NPROC / 6))
CRONJOB_TIMING="9 4 * * *"
;;
liquid)
DAEMON=elements
NETWORK=liquid
FEATURES=liquid
THREADS=$((NPROC / 6))
CRONJOB_TIMING="12 4 * * *"
;;
liquidtestnet)
DAEMON=elements
NETWORK=liquidtestnet
FEATURES=liquid
THREADS=$((NPROC / 6))
CRONJOB_TIMING="17 4 * * *"
;;
*)
echo "Usage: $0 (mainnet|testnet|testnet4|signet|liquid|liquidtestnet)"
echo "${USAGE}"
exit 1
;;
esac

# Run the popular address txt file generator before each run
POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"

# This function runs the job for generating the popular scripts text file for the precache arg
generate_popular_scripts() {
mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
rm -f "${POPULAR_SCRIPTS_FILE_RAW}" "${POPULAR_SCRIPTS_FILE}"

## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
## Only pick up addresses with 101 history events or more
## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
## It prints out progress to STDERR
echo "[*] Generating popular-scripts using ${THREADS} threads..."
cd "${HOME}/electrs"
HIGH_USAGE_THRESHOLD=101 \
JOB_THREAD_COUNT=${THREADS} \
nice cargo run \
--release \
--bin popular-scripts \
--features "${FEATURES}" \
-- \
--network "${NETWORK}" \
--db-dir "${DB_FOLDER}" \
> "${POPULAR_SCRIPTS_FILE_RAW}"

## Sorted and deduplicated just in case
sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
rm "${POPULAR_SCRIPTS_FILE_RAW}"
}

# This function is for inserting the cronjob for generating the popular scripts
CRONJOB_CMD="\"${HOME}/electrs/start\" \"${NETWORK}\" popular-scripts"
insert_cronjob() {
(crontab -l 2>/dev/null; echo "${CRONJOB_TIMING} ${CRONJOB_CMD}") | crontab -
}

case "${2}" in
popular-scripts)
echo "[*] Only generate popular-scripts, then exit"
generate_popular_scripts
exit 0
;;
"")
# If the 2nd arg isn't passed, just run the normal electrs script as-is
;;
*)
echo "${USAGE}"
exit 1
;;
esac
Expand Down Expand Up @@ -115,31 +175,13 @@ do
ELECTRUM_TXS_LIMIT=9000
fi

# Run the popular address txt file generator before each run
POPULAR_SCRIPTS_FOLDER="${HOME}/popular-scripts/${NETWORK}"
POPULAR_SCRIPTS_FILE_RAW="${POPULAR_SCRIPTS_FOLDER}/popular-scripts-raw.txt"
POPULAR_SCRIPTS_FILE="${POPULAR_SCRIPTS_FOLDER}/popular-scripts.txt"
mkdir -p "${POPULAR_SCRIPTS_FOLDER}"
rm -f "${POPULAR_SCRIPTS_FILE_RAW}" "${POPULAR_SCRIPTS_FILE}"

## Use nproc * 4 threads to generate the txt file (lots of iowait, so 2x~4x core count is ok)
## Only pick up addresses with 101 history events or more
## (Without lowering MIN_HISTORY_ITEMS_TO_CACHE this is the lowest we can go)
## It prints out progress to STDERR
echo "[*] Generating popular-scripts using ${THREADS} threads..."
HIGH_USAGE_THRESHOLD=101 \
JOB_THREAD_COUNT=${THREADS} \
cargo run \
--release \
--bin popular-scripts \
--features "${FEATURES}" \
-- \
--network "${NETWORK}" \
--db-dir "${DB_FOLDER}" \
> "${POPULAR_SCRIPTS_FILE_RAW}"
# if [ ! -e "${POPULAR_SCRIPTS_FILE}" ];then
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you commented out the wrong thing.

You left the "add crontab" if block

But commented out the "perform popular_scripts job" if block

If you run this, it will add the cron job but not run the actual job when the file is missing, and the electrs run command will fail when it tries to run the cache operations on a file that doesn't exist.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, switching the if block you comment out will change the behavior from

"Always run the popular_script job before running electrs" (current)

to

"Only run the popular_script job when the file doesn't exist. Otherwise, the cron job should have kept the file up to date for us so we shouldn't need to run it" (This PR)

# generate_popular_scripts
# fi

## Sorted and deduplicated just in case
sort "${POPULAR_SCRIPTS_FILE_RAW}" | uniq > "${POPULAR_SCRIPTS_FILE}"
if [ ! (crontab -l | grep "${CRONJOB_CMD}") ];then
insert_cronjob
fi

# Run the electrs process (Note: db-dir is used in both commands)
cargo run \
Expand Down
Loading