Merge branch 'master' into accounthistorycount-filters

.github/workflows/fullsync-tests.yml

@@ -5,12 +5,16 @@ on:
       - master
       - epic/*
     types: [labeled, opened, reopened, synchronize]
+  workflow_dispatch:
+    inputs:
+      block_ranges:
+        description: 'Optionally restrict to specific block ranges. Should be formatted as multiple start block, such as "350000 50000 750000". Defaults to all block ranges'
+        required: false
 
 jobs:
   build-binaries:
-    if: contains(github.event.pull_request.labels.*.name, 'ci/sync')
-    runs-on: [self-hosted, linux, x64, server-2]
-    # runs-on: ubuntu-18.04
+    if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch'
+    runs-on: [self-hosted, linux, x64, builder]
     steps:
     - uses: actions/checkout@v3
       with:
@@ -43,7 +47,7 @@ jobs:
           ci/parallel_sync/sync_then_diff.sh
 
   generate-matrix:
-      if: contains(github.event.pull_request.labels.*.name, 'ci/sync')
+      if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch'
       runs-on: [self-hosted, linux, x64]
 
       # Add "id-token" with the intended permissions.
@@ -68,16 +72,20 @@ jobs:
           uses: 'google-github-actions/setup-gcloud@v0'
 
         - id: set-matrix
+          name: 'Set matrix output'
           run: |
-            SNAPSHOTS=$(gsutil ls gs://team-drop/${{github.base_ref}}-datadir)
-            BLOCKS=$(echo "$SNAPSHOTS" | sed -e 's/.*\-\(.*\)\.tar.*/\1/' | grep -v gs | sort -n | head -n -1)
-            JSON=$(jq -n -c -M --arg blocks "$BLOCKS" '{blocks: ($blocks | split("\n") | .[] |= tonumber | to_entries | map({start: .value , stop: (.value + 50000)}))}')
-            echo "::set-output name=matrix::$JSON"
+            if [ -n "${{ inputs.block_ranges }}" ]; then
+              BLOCKS=$(echo ${{ inputs.block_ranges }} | tr ' ' '\n')
+            else
+              SNAPSHOTS=$(gsutil ls gs://team-drop/master-datadir)
+              BLOCKS=$(echo "$SNAPSHOTS" | sed -e 's/.*\-\(.*\)\.tar.*/\1/' | grep -v gs | sort -n | head -n -1)
+            fi
+            MATRIX_JSON=$(jq -n -c -M --arg blocks "$BLOCKS" '{blocks: ($blocks | split("\n") | .[] |= tonumber | to_entries | map({start: .value , stop: (.value + 50000)}))}')
+            echo "MATRIX=$MATRIX_JSON" >> $GITHUB_OUTPUT
 
   sync:
     runs-on: [self-hosted, linux, x64]
     needs: [build-binaries, generate-matrix]
-    if: contains(github.event.pull_request.labels.*.name, 'ci/sync')
 
     strategy:
       matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
@@ -89,18 +97,15 @@ jobs:
       START_BLOCK: ${{matrix.blocks.start}}
       DEFID_BIN: ./defid
       DEFI_CLI_BIN: ./defi-cli
-      REF_LOG_DIR: ${{github.base_ref}}-datadir/log
-      BASE_REF: ${{github.base_ref}}
+      REF_LOG_DIR: master-datadir/log
+      BASE_REF: master
     timeout-minutes: 4320
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Install Dependencies
-      run: sudo apt-get update && sudo apt-get install -y wget
-
     - name: Download Snapshot
-      run: wget https://storage.googleapis.com/team-drop/${{github.base_ref}}-datadir/datadir-${{matrix.blocks.start}}.tar.gz
+      run: aria2c -x16 -s16 https://storage.googleapis.com/team-drop/master-datadir/datadir-${{matrix.blocks.start}}.tar.gz
 
     - name: Create datadir
       run: mkdir $DATADIR && tar -C $DATADIR -xvf datadir-${{matrix.blocks.start}}.tar.gz

src/dbwrapper.cpp

@@ -14,6 +14,8 @@
 #include <stdint.h>
 #include <algorithm>
 
+bool levelDBChecksum{true};
+
 class CDefiLevelDBLogger : public leveldb::Logger {
 public:
     // This code is adapted from posix_logger.h, which is why it is using vsprintf.
@@ -105,7 +107,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
         v++;
         return v;
     };
-    
+
     leveldb::Options options;
     options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
     options.write_buffer_size = ceil_power_of_two(std::min(static_cast<size_t>(64)
@@ -132,6 +134,10 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo
     syncoptions.sync = true;
     options = GetOptions(nCacheSize);
     options.create_if_missing = true;
+
+    readoptions.verify_checksums = levelDBChecksum;
+    iteroptions.verify_checksums = levelDBChecksum;
+
     if (fMemory) {
         penv = leveldb::NewMemEnv(leveldb::Env::Default());
         options.env = penv;

src/dbwrapper.h

@@ -18,6 +18,9 @@
 
 static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
 static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
+static const std::string DEFAULT_LEVELDB_CHECKSUM = "auto";
+
+extern bool levelDBChecksum;
 
 class dbwrapper_error : public std::runtime_error
 {

src/dummywallet.cpp

@@ -49,6 +49,7 @@ void DummyWalletInit::AddWalletOptions() const
         "-flushwallet",
         "-privdb",
         "-walletrejectlongchains",
+        "-backupwallet",
     });
 }
 

src/init.cpp

@@ -944,6 +944,21 @@ void InitParameterInteraction()
         if (gArgs.SoftSetBoolArg("-whitelistrelay", true))
             LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> setting -whitelistrelay=1\n", __func__);
     }
+
+    // Parse leveldb checksum
+    const auto checksumArg = gArgs.GetArg("-leveldbchecksum", DEFAULT_LEVELDB_CHECKSUM);
+    if (checksumArg == "true"){
+        levelDBChecksum = true;
+    } else if (checksumArg == "false") {
+        levelDBChecksum = false;
+    } else {
+        if (checksumArg != "auto"){
+            InitWarning("Invalid value for -leveldbchecksum, setting default value -> 'auto'");
+        }
+        if (levelDBChecksum = gArgs.IsArgSet("-masternode_operator"); levelDBChecksum) {
+            LogPrintf("%s: parameter interaction: -masternode_operator -> setting -leveldbchecksum='true'\n", __func__);
+        }
+    }
 }
 
 /**