From 2be32a4214f792340fdbbd19769dba7727fe51ef Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 20:44:36 -0600 Subject: [PATCH 1/9] ci: Consolidate GitHub workflows for Qt and test coverage This commit consolidates the former Actions into a new GitHub Actions workflow that builds and tests DigiByte Core on Ubuntu with full Qt support and test coverage enabled. The workflow: - Runs on Ubuntu latest - Installs required build dependencies - Sets up Python packages needed for testing - Configures Qt5 development tools This helps ensure code quality by running comprehensive tests on pull requests develop/master branches. --- .../ci-coverage-mac-full-qt-tests.yml | 48 ----------------- .../ci-coverage-mac-no-tests-no-qt.yml | 53 ------------------- .../ci-coverage-ubuntu-full-qt-tests.yml | 30 ++++++++--- .../ci-coverage-ubuntu-no-tests-no-qt.yml | 44 --------------- .vscode/settings.json | 12 +---- 5 files changed, 24 insertions(+), 163 deletions(-) delete mode 100644 .github/workflows/ci-coverage-mac-full-qt-tests.yml delete mode 100644 .github/workflows/ci-coverage-mac-no-tests-no-qt.yml delete mode 100644 .github/workflows/ci-coverage-ubuntu-no-tests-no-qt.yml diff --git a/.github/workflows/ci-coverage-mac-full-qt-tests.yml b/.github/workflows/ci-coverage-mac-full-qt-tests.yml deleted file mode 100644 index 34aec6d0b0..0000000000 --- a/.github/workflows/ci-coverage-mac-full-qt-tests.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: 'Build & Check: macOS Full QT & Tests' - -on: [push] - -jobs: - build: - runs-on: macos-10.15 - strategy: - matrix: - os: [x86_64-mac] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Brew Install Dependencies - run: brew install automake pandoc - - - name: Dependencies - run: | - cd depends - make -j3 - cd .. - - - name: Install pypandoc - run: pip3 install pypandoc - - - name: Install scrypt for Python3 - run: pip3 install digibyte_scrypt - - - name: Auto Generate - run: ./autogen.sh - - - name: Configure - run: CONFIG_SITE=$PWD/depends/x86_64-apple-darwin19.6.0/share/config.site ./configure - - - name: Make - run: make -j3 - - - name: Make Check - run: make check - - - name: Upload Test Suite Log - uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-suite-log - path: /src/test-suite.log \ No newline at end of file diff --git a/.github/workflows/ci-coverage-mac-no-tests-no-qt.yml b/.github/workflows/ci-coverage-mac-no-tests-no-qt.yml deleted file mode 100644 index 9cbb8fe36a..0000000000 --- a/.github/workflows/ci-coverage-mac-no-tests-no-qt.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: 'Build & Check: macOS No QT & No Tests' - -on: [push] - -jobs: - build: - runs-on: macos-10.15 - strategy: - matrix: - os: [x86_64-mac] - - steps: - - uses: actions/checkout@v2 - - - name: Brew Install Dependencies - run: brew install automake boost@1.76 berkeley-db@4 qt@5 miniupnpc libnatpmp zeromq librsvg openssl pandoc - - - name: Brew Link OpenSSL - run: brew link openssl --force - - - name: Brew Link Boost - run: brew link boost@1.76 --force - - - name: Brew Link BDB - run: brew link berkeley-db@4 --force - - - name: Brew Force Link Openssl - run: export LDFLAGS="-L/usr/local/opt/openssl/lib -L/usr/local/lib -L/usr/local/opt/expat/lib" && export CFLAGS="-I/usr/local/opt/openssl/include/ -I/usr/local/include -I/usr/local/opt/expat/include" && export CPPFLAGS="-I/usr/local/opt/openssl/include/ -I/usr/local/include -I/usr/local/opt/expat/include" - - - name: Install pypandoc - run: pip3 install pypandoc - - - name: Install scrypt for Python3 - run: pip3 install digibyte_scrypt - - - name: Auto Generate - run: ./autogen.sh - - - name: Configure - run: ./configure --without-gui --disable-tests - - - name: Make - run: make -j3 - - - name: Make Check - run: make check - - - name: Upload Test Suite Log - uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-suite-log - path: /src/test-suite.log \ No newline at end of file diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index 44b2a42c49..c2d4f58064 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -1,6 +1,9 @@ name: 'Build & Check: Ubuntu Full QT & Tests' -on: [push] +on: + push: + pull_request: + branches: [ develop, master ] jobs: build: @@ -12,29 +15,42 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Get Dependencies - run: sudo apt update && sudo apt-get install build-essential libtool libssl-dev autotools-dev automake pkg-config bsdmainutils python3 libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libsqlite3-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev software-properties-common pandoc + - name: Install Basic Build Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential libtool autotools-dev automake pkg-config python3 \ + libssl-dev bsdmainutils libevent-dev \ + libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev \ + libsqlite3-dev \ + libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools \ + libqrencode-dev - name: Install pypandoc run: pip3 install pypandoc - + - name: Install scrypt for Python3 run: pip3 install digibyte_scrypt - + - name: Install BerkleyDB run: ./contrib/install_db4.sh `pwd` --enable-cxx + - name: Build Dependencies + run: | + cd depends + make -j3 HOST=x86_64-linux-gnu + cd .. + - name: Auto Generate run: ./autogen.sh - name: Configure - run: export BDB_PREFIX="${PWD}/db4" && ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" + run: CONFIG_SITE=$PWD/depends/x86_64-linux-gnu/share/config.site ./configure --with-gui=yes - name: Make run: make -j3 - name: Make Check - run: sudo make check + run: make check -j3 - name: Upload Test Suite Log uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-coverage-ubuntu-no-tests-no-qt.yml b/.github/workflows/ci-coverage-ubuntu-no-tests-no-qt.yml deleted file mode 100644 index 811f191045..0000000000 --- a/.github/workflows/ci-coverage-ubuntu-no-tests-no-qt.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: 'Build & Check: No QT & No Tests' - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - os: [x86_64-linux-gnu] - - steps: - - uses: actions/checkout@v2 - - - name: Get Dependencies - run: sudo apt update && sudo apt-get install build-essential libtool libssl-dev autotools-dev automake pkg-config bsdmainutils python3 libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libsqlite3-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev software-properties-common pandoc - - - name: Install pypandoc - run: pip3 install pypandoc - - - name: Install scrypt for Python3 - run: pip3 install digibyte_scrypt - - - name: Install BerkleyDB - run: ./contrib/install_db4.sh `pwd` --enable-cxx - - - name: Auto Generate - run: ./autogen.sh - - - name: Configure - run: export BDB_PREFIX="${PWD}/db4" && ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" --without-gui --disable-tests - - - name: Make - run: make -j3 - - - name: Make Check - run: make check - - - name: Upload Test Suite Log - uses: actions/upload-artifact@v3 - if: failure() - with: - name: test-suite-log - path: /src/test-suite.log \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index e525a3318f..3316f41f39 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,15 +27,5 @@ "typeindex": "cpp", "ios": "cpp" }, - "cmake.ignoreCMakeListsMissing": true, - "boost-test-adapter-robaho.tests": [ - { - "testExecutables": [ - { - "glob": "**/*{_test,_test.exe}" - } - ], - "debugConfig": "Test Config" - } - ], + "cmake.ignoreCMakeListsMissing": true } \ No newline at end of file From 55a7a42f5b09b87658ba3a0fee13618b8a696655 Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 20:45:59 -0600 Subject: [PATCH 2/9] ci: Added missing configure options. --- .github/workflows/ci-coverage-ubuntu-full-qt-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index c2d4f58064..ab5f4451dd 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -44,7 +44,7 @@ jobs: run: ./autogen.sh - name: Configure - run: CONFIG_SITE=$PWD/depends/x86_64-linux-gnu/share/config.site ./configure --with-gui=yes + run: CONFIG_SITE=$PWD/depends/x86_64-linux-gnu/share/config.site ./configure --with-bdb --with-sqlite --with-gui=yes - name: Make run: make -j3 From 7b9372b68764a28c8a70bd19d6daf5edd9308d54 Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 21:36:40 -0600 Subject: [PATCH 3/9] ci: Simplify CI workflow configuration and improve documentation formatting --- .../ci-coverage-ubuntu-full-qt-tests.yml | 3 - doc/build-unix.md | 83 ++++++++----------- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index ab5f4451dd..6e3e0c659b 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -8,9 +8,6 @@ on: jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - os: [x86_64-linux-gnu] steps: - uses: actions/checkout@v2 diff --git a/doc/build-unix.md b/doc/build-unix.md index 5211e03530..393a39a6e6 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -1,11 +1,11 @@ -UNIX BUILD NOTES -==================== +# UNIX BUILD NOTES + Some notes on how to build DigiByte Core in Unix. (For BSD specific instructions, see `build-*bsd.md` in this directory.) -Note ---------------------- +## Note + Always use absolute paths to configure and compile DigiByte Core and the dependencies. For example, when specifying the path of the dependency: @@ -14,20 +14,18 @@ For example, when specifying the path of the dependency: Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures the usage of the absolute path. -To Build ---------------------- +## To Build -```bash -./autogen.sh -./configure -make # use "-j N" for N parallel jobs -make install # optional -``` + ```bash + ./autogen.sh + ./configure + make # use "-j N" for N parallel jobs + make install # optional + ``` This will build digibyte-qt as well, if the dependencies are met. -Dependencies ---------------------- +### Dependencies These dependencies are required: @@ -51,14 +49,12 @@ Optional dependencies: For the versions used, see [dependencies.md](dependencies.md) -Memory Requirements --------------------- +### Memory Requirements C++ compilers are memory-hungry. It is recommended to have at least 1.5 GB of memory available when compiling DigiByte Core. On systems with less, gcc can be tuned to conserve memory with additional CXXFLAGS: - ./configure CXXFLAGS="--param ggc-min-expand=1 --param ggc-min-heapsize=32768" Alternatively, or in addition, debugging information can be skipped for compilation. The default compile flags are @@ -74,7 +70,7 @@ Finally, clang (often less resource hungry) can be used instead of gcc, which is ### Ubuntu & Debian -#### Dependency Build Instructions +#### Ubuntu & Debian Dependency Build Instructions Build requirements: @@ -124,10 +120,9 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a digibyte-qt executable will be built by default. - ### Fedora -#### Dependency Build Instructions +#### Fedora Dependency Build Instructions Build requirements: @@ -179,13 +174,12 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a digibyte-qt executable will be built by default. -Notes ------ +## Notes + The release is built with GCC and then "strip digibyted" to strip the debug symbols, which reduces the executable size by about 90%. -miniupnpc ---------- +## miniupnpc [miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and @@ -195,8 +189,7 @@ turned off by default. See the configure options for UPnP behavior desired: --disable-upnp-default (the default) UPnP support turned off by default at runtime --enable-upnp-default UPnP support turned on by default at runtime -libnatpmp ---------- +## libnatpmp [libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and @@ -206,15 +199,15 @@ turned off by default. See the configure options for NAT-PMP behavior desired: --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime --enable-natpmp-default NAT-PMP support turned on by default at runtime -Berkeley DB ------------ +## Berkeley DB + It is recommended to use Berkeley DB 4.8. If you have to build it yourself, you can use [the installation script included in contrib/](/contrib/install_db4.sh) like so: -```shell -./contrib/install_db4.sh `pwd` -``` + ```shell + ./contrib/install_db4.sh `pwd` + ``` from the root of the repository. @@ -222,17 +215,16 @@ Otherwise, you can build DigiByte Core from self-compiled [depends](/depends/REA **Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). -Boost ------ +## Boost + If you need to build Boost yourself: sudo su ./bootstrap.sh ./bjam install +## Security -Security --------- To help make your DigiByte Core installation more secure by making certain attacks impossible to exploit even if a vulnerability is found, binaries are hardened by default. This can be disabled with: @@ -242,9 +234,9 @@ Hardening Flags: ./configure --enable-hardening ./configure --disable-hardening - Hardening enables the following features: -* _Position Independent Executable_: Build position independent code to take advantage of Address Space Layout Randomization + +* *Position Independent Executable*: Build position independent code to take advantage of Address Space Layout Randomization offered by some kernels. Attackers who can cause execution of code at an arbitrary memory location are thwarted if they don't know where anything useful is located. The stack and heap are randomly located by default, but this allows the code section to be @@ -262,7 +254,7 @@ Hardening enables the following features: TYPE ET_DYN -* _Non-executable Stack_: If the stack is executable then trivial stack-based buffer overflow exploits are possible if +* *Non-executable Stack*: If the stack is executable then trivial stack-based buffer overflow exploits are possible if vulnerable buffers are found. By default, DigiByte Core should be built with a non-executable stack, but if one of the libraries it uses asks for an executable stack or someone makes a mistake and uses a compiler extension which requires an executable stack, it will silently build an @@ -277,8 +269,8 @@ Hardening enables the following features: The STK RW- means that the stack is readable and writeable but not executable. -Disable-wallet mode --------------------- +## Disable-wallet mode + When the intention is to run only a P2P node without a wallet, DigiByte Core may be compiled in disable-wallet mode with: @@ -288,15 +280,14 @@ In this case there is no dependency on Berkeley DB 4.8 and SQLite. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. -Additional Configure Flags --------------------------- +## Additional Configure Flags + A list of additional configure flags can be displayed with: ./configure --help +## Setup and Build Example: Arch Linux -Setup and Build Example: Arch Linux ------------------------------------ This example lists the steps necessary to setup and build a command line only, non-wallet distribution of the latest changes on Arch Linux: pacman -S git base-devel boost libevent python @@ -313,9 +304,8 @@ or building and depending on a local version of Berkeley DB 4.8. The readily ava As mentioned above, when maintaining portability of the wallet between the standard DigiByte Core distributions and independently built node software is desired, Berkeley DB 4.8 must be used. +## ARM Cross-compilation -ARM Cross-compilation -------------------- These steps can be performed on, for example, an Ubuntu VM. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. @@ -334,5 +324,4 @@ To build executables for ARM: CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++ make - For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. From 4b2c7bb06dd014207308324302458a827a74f17f Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 21:58:21 -0600 Subject: [PATCH 4/9] ci: Enhance caching for APT packages, pip, and build dependencies in CI workflow --- .../ci-coverage-ubuntu-full-qt-tests.yml | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index 6e3e0c659b..f6fb3f5510 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -11,7 +11,29 @@ jobs: steps: - uses: actions/checkout@v2 - + + - name: Cache APT Packages + uses: actions/cache@v3 + with: + path: | + /var/cache/apt + /var/lib/apt/lists + key: v1-${{ runner.os }}-apt-${{ hashFiles('**/ci-coverage-ubuntu-full-qt-tests.yml') }} + restore-keys: | + v1-${{ runner.os }}-apt- + v1-${{ runner.os }}- + + - name: Cache pip + uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + ~/.cache/pip3 + key: v1-${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + v1-${{ runner.os }}-pip- + v1-${{ runner.os }}- + - name: Install Basic Build Dependencies run: | sudo apt-get update @@ -31,6 +53,17 @@ jobs: - name: Install BerkleyDB run: ./contrib/install_db4.sh `pwd` --enable-cxx + - name: Cache depends build + uses: actions/cache@v3 + with: + path: | + depends/built + depends/sdk-sources + depends/x86_64-linux-gnu + key: v1-${{ runner.os }}-depends-${{ hashFiles('depends/Makefile') }} + restore-keys: | + v1-${{ runner.os }}-depends- + - name: Build Dependencies run: | cd depends From b9fa6b0e867d64b63b13f514f45becae2f0ea579 Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 22:10:29 -0600 Subject: [PATCH 5/9] ci: Refactor CI workflow to separate dependencies installation and build steps --- .../ci-coverage-ubuntu-full-qt-tests.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index f6fb3f5510..4f9af16e44 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -6,9 +6,8 @@ on: branches: [ develop, master ] jobs: - build: + dependencies: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v2 @@ -44,12 +43,11 @@ jobs: libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools \ libqrencode-dev - - name: Install pypandoc - run: pip3 install pypandoc + - name: Install Python Packages + run: | + pip3 install pypandoc + pip3 install digibyte_scrypt - - name: Install scrypt for Python3 - run: pip3 install digibyte_scrypt - - name: Install BerkleyDB run: ./contrib/install_db4.sh `pwd` --enable-cxx @@ -70,6 +68,12 @@ jobs: make -j3 HOST=x86_64-linux-gnu cd .. + build: + needs: dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Auto Generate run: ./autogen.sh From 9d53a9cfb9abc101f5cc9138cc2f7ae68a65bbd1 Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 22:22:13 -0600 Subject: [PATCH 6/9] ci: Refactor CI workflow to consolidate build steps and enhance caching --- .../ci-coverage-ubuntu-full-qt-tests.yml | 82 ++++++++----------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index 4f9af16e44..58d8599544 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -6,22 +6,11 @@ on: branches: [ develop, master ] jobs: - dependencies: + build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache APT Packages - uses: actions/cache@v3 - with: - path: | - /var/cache/apt - /var/lib/apt/lists - key: v1-${{ runner.os }}-apt-${{ hashFiles('**/ci-coverage-ubuntu-full-qt-tests.yml') }} - restore-keys: | - v1-${{ runner.os }}-apt- - v1-${{ runner.os }}- - - name: Cache pip uses: actions/cache@v3 with: @@ -29,11 +18,30 @@ jobs: ~/.cache/pip ~/.cache/pip3 key: v1-${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: v1-${{ runner.os }}-pip- + + - name: Cache BerkeleyDB + uses: actions/cache@v3 + with: + path: db4 + key: v1-${{ runner.os }}-db4-${{ hashFiles('contrib/install_db4.sh') }} + restore-keys: v1-${{ runner.os }}-db4- + + - name: Cache depends build + uses: actions/cache@v3 + with: + path: | + depends/built + depends/sources + depends/sdk-sources + depends/work + depends/x86_64-linux-gnu + key: v1-${{ runner.os }}-depends-${{ hashFiles('depends/Makefile') }}-${{ github.sha }} restore-keys: | - v1-${{ runner.os }}-pip- - v1-${{ runner.os }}- + v1-${{ runner.os }}-depends-${{ hashFiles('depends/Makefile') }}- + v1-${{ runner.os }}-depends- - - name: Install Basic Build Dependencies + - name: Install Build Dependencies run: | sudo apt-get update sudo apt-get install -y build-essential libtool autotools-dev automake pkg-config python3 \ @@ -45,22 +53,13 @@ jobs: - name: Install Python Packages run: | - pip3 install pypandoc - pip3 install digibyte_scrypt - - - name: Install BerkleyDB - run: ./contrib/install_db4.sh `pwd` --enable-cxx + pip3 install pypandoc digibyte_scrypt - - name: Cache depends build - uses: actions/cache@v3 - with: - path: | - depends/built - depends/sdk-sources - depends/x86_64-linux-gnu - key: v1-${{ runner.os }}-depends-${{ hashFiles('depends/Makefile') }} - restore-keys: | - v1-${{ runner.os }}-depends- + - name: Install BerkeleyDB + run: | + if [ ! -d "db4" ]; then + ./contrib/install_db4.sh `pwd` --enable-cxx + fi - name: Build Dependencies run: | @@ -68,22 +67,13 @@ jobs: make -j3 HOST=x86_64-linux-gnu cd .. - build: - needs: dependencies - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Auto Generate - run: ./autogen.sh - - - name: Configure - run: CONFIG_SITE=$PWD/depends/x86_64-linux-gnu/share/config.site ./configure --with-bdb --with-sqlite --with-gui=yes - - - name: Make - run: make -j3 + - name: Configure and Build + run: | + ./autogen.sh + CONFIG_SITE=$PWD/depends/x86_64-linux-gnu/share/config.site ./configure --with-bdb --with-sqlite --with-gui=yes + make -j3 - - name: Make Check + - name: Run Tests run: make check -j3 - name: Upload Test Suite Log @@ -91,4 +81,4 @@ jobs: if: failure() with: name: test-suite-log - path: /src/test-suite.log \ No newline at end of file + path: test-suite.log \ No newline at end of file From aa92cd2834e4db9566f33d5c9267a55cfe062d32 Mon Sep 17 00:00:00 2001 From: GTO90 Date: Mon, 16 Dec 2024 22:28:40 -0600 Subject: [PATCH 7/9] ci: Enhance caching for pip, BerkeleyDB, and build dependencies in CI workflow --- .github/workflows/ci-coverage-ubuntu-full-qt-tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index 58d8599544..3862948dc8 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -12,6 +12,7 @@ jobs: - uses: actions/checkout@v2 - name: Cache pip + id: cache-pip uses: actions/cache@v3 with: path: | @@ -21,6 +22,7 @@ jobs: restore-keys: v1-${{ runner.os }}-pip- - name: Cache BerkeleyDB + id: cache-db4 uses: actions/cache@v3 with: path: db4 @@ -28,6 +30,7 @@ jobs: restore-keys: v1-${{ runner.os }}-db4- - name: Cache depends build + id: cache-depends uses: actions/cache@v3 with: path: | @@ -52,16 +55,17 @@ jobs: libqrencode-dev - name: Install Python Packages + if: steps.cache-pip.outputs.cache-hit != 'true' run: | pip3 install pypandoc digibyte_scrypt - name: Install BerkeleyDB + if: steps.cache-db4.outputs.cache-hit != 'true' run: | - if [ ! -d "db4" ]; then - ./contrib/install_db4.sh `pwd` --enable-cxx - fi + ./contrib/install_db4.sh `pwd` --enable-cxx - name: Build Dependencies + if: steps.cache-depends.outputs.cache-hit != 'true' run: | cd depends make -j3 HOST=x86_64-linux-gnu From e51b43af98888c1ae38c6a05793746430aa3af0a Mon Sep 17 00:00:00 2001 From: GTO90 Date: Tue, 17 Dec 2024 09:35:26 -0600 Subject: [PATCH 8/9] ci: Update CI workflow to include python3-zmq dependency and improve README formatting --- .../ci-coverage-ubuntu-full-qt-tests.yml | 4 +- ci/README.md | 28 ++++---- qa/README.md | 72 +++++++++++-------- 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index 3862948dc8..eee92451f8 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -52,12 +52,12 @@ jobs: libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev \ libsqlite3-dev \ libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools \ - libqrencode-dev + libqrencode-dev python3-zmq - name: Install Python Packages if: steps.cache-pip.outputs.cache-hit != 'true' run: | - pip3 install pypandoc digibyte_scrypt + pip3 install pypandoc digibyte_scrypt pyzmq - name: Install BerkeleyDB if: steps.cache-db4.outputs.cache-hit != 'true' diff --git a/ci/README.md b/ci/README.md index 3c5f04c39e..afea0d9285 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,8 +1,8 @@ -## CI Scripts +# CI Scripts This directory contains scripts for each build step in each build stage. -### Running a Stage Locally +## Running a Stage Locally Be aware that the tests will be built and run in-place, so please run at your own risk. If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. @@ -17,21 +17,21 @@ system in a virtual machine with a Linux operating system of your choice. To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage requires `docker` to be installed. To install all requirements on Ubuntu, run -``` -sudo apt install docker.io bash -``` + ```bash + sudo apt install docker.io bash + ``` To run the default test stage, -``` -./ci/test_run_all.sh -``` + ```bash + ./ci/test_run_all.sh + ``` To run the test stage with a specific configuration, -``` -FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh -``` + ```bash + FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh + ``` ### Configurations @@ -51,9 +51,9 @@ is used as the default configuration with fallback values. It is also possible to force a specific configuration without modifying the file. For example, -``` -MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh -``` + ```bash + MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh + ``` The files starting with `0n` (`n` greater than 0) are the scripts that are run in order. diff --git a/qa/README.md b/qa/README.md index 6ff4e85d6e..73b3c6f285 100644 --- a/qa/README.md +++ b/qa/README.md @@ -1,3 +1,5 @@ +# Testing + The [pull-tester](/qa/pull-tester/) folder contains a script to call multiple tests from the [rpc-tests](/qa/rpc-tests/) folder. @@ -5,41 +7,49 @@ Every pull request to the digibyte repository is built and run through the regression test suite. You can also run all or only individual tests locally. -Test dependencies -================= +## Test dependencies + Before running the tests, the following must be installed. -Unix ----- +## Unix + The python3-zmq library is required. On Ubuntu or Debian it can be installed via: -``` -sudo apt-get install python3-zmq -``` -OS X ------- -``` -pip3 install pyzmq -``` + ```bash + sudo apt-get install python3-zmq + ``` -Running tests -============= +## OS X + + ```bash + pip3 install pyzmq + ``` + +## Running tests You can run any single test by calling + ```bash qa/pull-tester/rpc-tests.py + ``` Or you can run any combination of tests by calling + ```bash qa/pull-tester/rpc-tests.py ... + ``` Run the regression test suite with + ```bash qa/pull-tester/rpc-tests.py + ``` Run all possible tests with + ```bash qa/pull-tester/rpc-tests.py -extended + ``` By default, tests will be run in parallel. To specify how many jobs to run, append `-parallel=n` (default n=4). @@ -48,17 +58,17 @@ If you want to create a basic coverage report for the rpc test suite, append `-- Possible options, which apply to each individual test run: -``` - -h, --help show this help message and exit - --nocleanup Leave digibyteds and test.* datadir on exit or error - --noshutdown Don't stop digibyteds after the test execution - --srcdir=SRCDIR Source directory containing digibyted/digibyte-cli - (default: ../../src) - --tmpdir=TMPDIR Root directory for datadirs - --tracerpc Print out all RPC calls as they are made - --coveragedir=COVERAGEDIR - Write tested RPC commands into this directory -``` + ```bash + -h, --help show this help message and exit + --nocleanup Leave digibyteds and test.* datadir on exit or error + --noshutdown Don't stop digibyteds after the test execution + --srcdir=SRCDIR Source directory containing digibyted/digibyte-cli + (default: ../../src) + --tmpdir=TMPDIR Root directory for datadirs + --tracerpc Print out all RPC calls as they are made + --coveragedir=COVERAGEDIR + Write tested RPC commands into this directory + ``` If you set the environment variable `PYTHON_DEBUG=1` you will get some debug output (example: `PYTHON_DEBUG=1 qa/pull-tester/rpc-tests.py wallet`). @@ -75,13 +85,13 @@ test state. If you get into a bad state, you should be able to recover with: -```bash -rm -rf cache -killall digibyted -``` + ```bash + rm -rf cache + killall digibyted + ``` + +## Writing tests -Writing tests -============= You are encouraged to write tests for new or existing features. Further information about the test framework and individual rpc tests is found in [qa/rpc-tests](/qa/rpc-tests). From ec5de94377c80352d80fda16229a493833648b5c Mon Sep 17 00:00:00 2001 From: GTO90 Date: Tue, 17 Dec 2024 09:43:36 -0600 Subject: [PATCH 9/9] ci: Remove push trigger from CI workflow for Ubuntu Full QT & Tests --- .github/workflows/ci-coverage-ubuntu-full-qt-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml index eee92451f8..b650bc9698 100644 --- a/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml +++ b/.github/workflows/ci-coverage-ubuntu-full-qt-tests.yml @@ -1,7 +1,6 @@ name: 'Build & Check: Ubuntu Full QT & Tests' on: - push: pull_request: branches: [ develop, master ]