diff --git a/.dockerignore b/.dockerignore index 2e663fa609..243e9f09c8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .code .cache +.ccache .idea # Editors. diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml index 18ab636c4d..a71e2e45ee 100644 --- a/.github/workflows/build-dev.yaml +++ b/.github/workflows/build-dev.yaml @@ -43,14 +43,27 @@ jobs: - name: Setup dependencies run: ./make.sh ci-setup-deps - + - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Restore cpp build cache + id: cpp-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build and package run: ./make.sh release @@ -61,6 +74,25 @@ jobs: name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} path: ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} + - name: Delete previous cpp build cache + if: ${{ github.ref == 'refs/heads/master' && steps.cpp-cache-restore.outputs.cache-hit }} + continue-on-error: true + run: | + gh extension install actions/gh-actions-cache + gh actions-cache delete "cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }}" --confirm + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Save cpp build cache + if: ${{ github.ref == 'refs/heads/master' }} + uses: actions/cache/save@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + docker-build: runs-on: ubuntu-latest needs: [build] @@ -69,18 +101,21 @@ jobs: steps: - uses: actions/checkout@v4 - - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars - - name: Download Binaries + - name: Download binaries uses: actions/download-artifact@v3 with: name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: ./build/ + + - name: Unpack binaries + run: tar -xvzf ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} -C ./build/ - name: Build defi image - run: rm .dockerignore && ./make.sh docker-defi-build + run: rm .dockerignore && ./make.sh docker-build-from-binaries - name: Login to Docker Hub uses: docker/login-action@v2 diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 6c4cba3507..dfd6a05953 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -35,11 +35,6 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 - with: - workspaces: lib -> ../build/lib/target - save-if: ${{ github.ref == 'refs/heads/master' }} - - name: Build and package run: ./make.sh release @@ -85,18 +80,21 @@ jobs: steps: - uses: actions/checkout@v4 - - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars - - name: Download Binaries + - name: Download binaries uses: actions/download-artifact@v3 with: name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: ./build/ + + - name: Unpack binaries + run: tar -xvzf ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} -C ./build/ - name: Build defi image - run: rm .dockerignore && ./make.sh docker-defi-build + run: rm .dockerignore && ./make.sh docker-build-from-binaries - name: Login to Docker Hub uses: docker/login-action@v2 diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index c75c626ca3..9c369a207f 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -33,11 +33,6 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 - with: - workspaces: lib -> ../build/lib/target - save-if: ${{ github.ref == 'refs/heads/master' }} - - name: Build and package run: ./make.sh release @@ -57,15 +52,18 @@ jobs: steps: - uses: actions/checkout@v4 - - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars - - name: Download Binaries + - name: Download binaries uses: actions/download-artifact@v3 with: name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: ./build/ + + - name: Unpack binaries + run: tar -xvzf ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} -C ./build/ - name: Build defi image - run: rm .dockerignore && ./make.sh docker-defi-build + run: rm .dockerignore && ./make.sh docker-build-from-binaries diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index da86d4ef83..c30a6d6a42 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -38,7 +38,7 @@ jobs: run: ./ci/extended_lint/main.sh rust: - name: Rust Lints + name: Rust Lint runs-on: ubuntu-latest container: defi/ain-builder:latest env: @@ -57,10 +57,12 @@ jobs: - name: Setup dependencies for user run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Rust lint cache + uses: Swatinem/rust-cache@v2 with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: lint - name: Build depends and configure run: ./make.sh build-deps && ./make.sh build-conf diff --git a/.github/workflows/tests-ethlibs.yaml b/.github/workflows/tests-ethlibs.yaml index 1bdf74a8f4..4f4b0223da 100644 --- a/.github/workflows/tests-ethlibs.yaml +++ b/.github/workflows/tests-ethlibs.yaml @@ -38,10 +38,23 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Restore cpp build cache + id: cpp-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build binaries run: ./make.sh build diff --git a/.github/workflows/tests-frontier.yml b/.github/workflows/tests-frontier.yml index fd389b0152..e47ce17e62 100644 --- a/.github/workflows/tests-frontier.yml +++ b/.github/workflows/tests-frontier.yml @@ -10,6 +10,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} +env: + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + jobs: build: runs-on: ubuntu-latest @@ -30,10 +34,23 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Restore cpp build cache + id: cpp-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build binaries run: ./make.sh build diff --git a/.github/workflows/tests-sync.yml b/.github/workflows/tests-sync.yml index 22fde03602..0bca0d17ee 100644 --- a/.github/workflows/tests-sync.yml +++ b/.github/workflows/tests-sync.yml @@ -21,6 +21,10 @@ concurrency: run-name: ${{ inputs.name || github.event.pull_request.title || github.ref_name }} +env: + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + jobs: build: if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' @@ -40,10 +44,23 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Restore cpp build cache + id: cpp-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build binaries run: ./make.sh build @@ -76,7 +93,6 @@ jobs: steps: - uses: actions/checkout@v4 - - run: git config --global --add safe.directory '*' - id: 'auth' name: 'Authenticate to Google Cloud' @@ -120,7 +136,6 @@ jobs: steps: - uses: actions/checkout@v4 - - run: git config --global --add safe.directory '*' - name: Download Snapshot run: aria2c -x16 -s16 https://storage.googleapis.com/team-drop/master-datadir/datadir-${{matrix.blocks.start}}.tar.gz @@ -128,7 +143,7 @@ jobs: - name: Create datadir run: mkdir $DATADIR && tar -C $DATADIR -xvf datadir-${{matrix.blocks.start}}.tar.gz - - name: Download Binaries + - name: Download binaries uses: actions/download-artifact@v3 with: name: defi-bins diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index faaf4ff257..b214875661 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -15,6 +15,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} +env: + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + jobs: e2e-tests: runs-on: ubuntu-latest @@ -45,10 +49,23 @@ jobs: - name: Setup user dependencies run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Restore cpp build cache + id: cpp-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ./build/depends + ./build/src + ~/.ccache + key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} + + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build binaries run: ./make.sh build @@ -76,10 +93,13 @@ jobs: - name: Setup dependencies for user run: ./make.sh ci-setup-user-deps - - uses: Swatinem/rust-cache@v2 + - name: Rust build cache + uses: Swatinem/rust-cache@v2 + id: rust-cache-restore with: workspaces: lib -> ../build/lib/target save-if: ${{ github.ref == 'refs/heads/master' }} + shared-key: ${{ env.TARGET }} - name: Build deps and configure run: ./make.sh build-deps && ./make.sh build-conf diff --git a/.gitignore b/.gitignore index 967415998d..968cdb071a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .code .cache +.ccache .idea # Editors. diff --git a/contrib/dockerfiles/defi.dockerfile b/contrib/dockerfiles/defi.dockerfile index 40db80d80a..301aca232d 100644 --- a/contrib/dockerfiles/defi.dockerfile +++ b/contrib/dockerfiles/defi.dockerfile @@ -2,14 +2,13 @@ ARG TARGET=x86_64-pc-linux-gnu FROM --platform=linux/amd64 ubuntu:latest as defi ARG TARGET -ARG PACKAGE +ARG BINARY_DIR ENV PATH=/app/bin:$PATH LABEL org.defichain.name="defichain" LABEL org.defichain.arch=${TARGET} WORKDIR /app -COPY ${PACKAGE} ./ -RUN tar -xvzf ${PACKAGE} --strip-components 1 +COPY ${BINARY_DIR} ./ RUN useradd --create-home defi && \ mkdir -p /data && \ diff --git a/make.sh b/make.sh index 832873a696..22c22d462d 100755 --- a/make.sh +++ b/make.sh @@ -18,6 +18,7 @@ setup_vars() { fi DOCKER_ROOT_CONTEXT=${DOCKER_ROOT_CONTEXT:-"."} + DOCKER_RELATIVE_BUILD_DIR=${DOCKER_RELATIVE_BUILD_DIR:-"./build"} DOCKERFILES_DIR=${DOCKERFILES_DIR:-"./contrib/dockerfiles"} DEFI_DOCKERFILE=${DEFI_DOCKERFILE:-"${DOCKERFILES_DIR}/defi.dockerfile"} @@ -320,24 +321,26 @@ docker_release() { _sign "$target" } -docker_defi_build() { +docker_build_from_binaries() { local target=${1:-${TARGET}} local img_prefix="${IMAGE_PREFIX}" local img_version="${IMAGE_VERSION}" - - local pkg_name="${img_prefix}-${img_version}-${target}.tar.gz" + local build_dir="${DOCKER_RELATIVE_BUILD_DIR}" local docker_context="${DOCKER_ROOT_CONTEXT}" local docker_file="${DEFI_DOCKERFILE}" - echo "> docker-defi-build"; + echo "> docker-build-from-binaries"; local img="${img_prefix}-${target}:${img_version}" echo "> building: ${img}" echo "> docker defi build: ${img}" + local versioned_name="${img_prefix}-${img_version}" + local versioned_build_dir="${build_dir}/${versioned_name}" + docker build -f "${docker_file}" \ - --build-arg PACKAGE="${pkg_name}" \ + --build-arg BINARY_DIR="${versioned_build_dir}" \ -t "${img}" "${docker_context}" } @@ -689,7 +692,7 @@ pkg_install_deps() { software-properties-common build-essential git libtool autotools-dev automake \ pkg-config bsdmainutils python3 python3-pip python3-venv libssl-dev libevent-dev libboost-system-dev \ libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev \ - libminiupnpc-dev libzmq3-dev libqrencode-dev wget \ + libminiupnpc-dev libzmq3-dev libqrencode-dev wget ccache \ libdb-dev libdb++-dev libdb5.3 libdb5.3-dev libdb5.3++ libdb5.3++-dev \ curl cmake zip unzip libc6-dev gcc-multilib locales locales-all @@ -743,6 +746,20 @@ pkg_install_deps_osx_tools() { _fold_end } +pkg_install_gh_cli() { + _fold_start "pkg-install-gh_cli" + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \ + dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ + chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) \ + signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] \ + https://cli.github.com/packages stable main" | \ + tee /etc/apt/sources.list.d/github-cli.list > /dev/null + apt-get update + apt-get install -y gh + +} + pkg_install_llvm() { _fold_start "pkg-install-llvm" # shellcheck disable=SC2086 @@ -1140,11 +1157,20 @@ _nproc() { # shellcheck disable=SC2129 ci_export_vars() { + local build_dir="${BUILD_DIR}" + if [[ -n "${GITHUB_ACTIONS-}" ]]; then # GitHub Actions echo "BUILD_VERSION=${IMAGE_VERSION}" >> "$GITHUB_ENV" echo "PATH=$HOME/.cargo/bin:$PATH" >> "$GITHUB_ENV" echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + + if [[ "${MAKE_DEBUG}" == "1" ]]; then + echo "BUILD_TYPE=debug" >> "$GITHUB_ENV" + else + echo "BUILD_TYPE=release" >> "$GITHUB_ENV" + fi + if [[ "${TARGET}" == "x86_64-w64-mingw32" ]]; then echo "PKG_TYPE=zip" >> "$GITHUB_ENV" else @@ -1158,6 +1184,7 @@ ci_setup_deps() { DEBIAN_FRONTEND=noninteractive pkg_install_deps DEBIAN_FRONTEND=noninteractive pkg_setup_locale DEBIAN_FRONTEND=noninteractive pkg_install_llvm + DEBIAN_FRONTEND=noninteractive pkg_install_gh_cli ci_setup_deps_target }