diff --git a/.dockerignore b/.dockerignore index 2e663fa609..243e9f09c8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .code .cache +.ccache .idea # Editors. diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml index de121cbf11..16d2a946f0 100644 --- a/.github/workflows/build-dev.yaml +++ b/.github/workflows/build-dev.yaml @@ -25,10 +25,14 @@ env: GIT_VERSION: 1 jobs: - linux-x64: + build: runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64-pc-linux-gnu, x86_64-w64-mingw32, x86_64-apple-darwin] env: - TARGET: x86_64-pc-linux-gnu + TARGET: ${{matrix.target}} + DOCKER_TARGET: builder steps: - uses: actions/checkout@v3 @@ -36,60 +40,70 @@ jobs: - name: Populate environment run: ./make.sh ci-export-vars - - name: Build and package - run: ./make.sh docker-release + - name: Cache docker depends build + id: docker-cache-build-deps + uses: actions/cache@v3 + with: + path: ./build/depends + key: docker-${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends - - name: Publish artifact - x86_64-pc-linux-gnu - uses: actions/upload-artifact@v3 + - name: Cache docker rust build + id: docker-cache-build-rust + uses: actions/cache@v3 with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz + path: ./build/lib + key: docker-${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust - - name: Login to Docker Hub - uses: docker/login-action@v2 + - name: Cache docker cpp build + id: docker-cache-build-cpp + uses: actions/cache@v3 with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} + path: ./build/src + key: docker-${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp - - name: Push to Docker Hub - run: | - set -e; ver=${{ env.BUILD_VERSION }} - docker tag defichain-x86_64-pc-linux-gnu:${ver} defi/defichain:${ver} - docker push defi/defichain:${ver} + - name: Cache docker ccache + id: docker-cache-ccache + uses: actions/cache@v3 + with: + path: ./build/.ccache + key: docker-${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache - win-x64: - runs-on: ubuntu-latest - env: - TARGET: x86_64-w64-mingw32 + - name: Use cache + if: | + ${{ steps.docker-cache-build-deps.outputs.cache-hit == 'true' }} && + ${{ steps.docker-cache-build-rust.outputs.cache-hit == 'true' }} && + ${{ steps.docker-cache-build-cpp.outputs.cache-hit == 'true' }} && + ${{ steps.docker-cache-ccache.outputs.cache-hit == 'true' }} + run: rm .dockerignore - steps: - - uses: actions/checkout@v3 + - name: Build builder image + run: ./make.sh docker-build - - name: Populate environment - run: ./make.sh ci-export-vars + - name: Build defi image + if: ${{ env.TARGET == 'x86_64-pc-linux-gnu' }} + run: find . -mmin -30 && DOCKER_TARGET="defi" IMAGE_NAME="defi/defichain:${{ env.BUILD_VERSION }}" ./make.sh docker-build - - name: Build and package - run: ./make.sh docker-release + - name: Deploy and package + run: | + ./make.sh docker-deploy + ./make.sh docker-deploy-build + ./make.sh package - - name: Publish artifact - x86_64-w64-mingw32 + - name: Publish artifact uses: actions/upload-artifact@v3 with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32 - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.tar.gz + name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} - osx-x64: - runs-on: ubuntu-latest - env: - TARGET: x86_64-apple-darwin - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release + - name: Login to Docker Hub + if: ${{ env.TARGET == 'x86_64-pc-linux-gnu' }} + uses: docker/login-action@v2 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Publish artifact - x86_64-apple-darwin - uses: actions/upload-artifact@v3 + - name: Push to Docker Hub + if: ${{ env.TARGET == 'x86_64-pc-linux-gnu' }} + run: | + set -e; ver=${{ env.BUILD_VERSION }} + docker push defi/defichain:${ver} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 69bea3660d..5fd93e96e9 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -12,88 +12,14 @@ env: GIT_VERSION: 1 jobs: - linux-x64: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - env: - TARGET: x86_64-pc-linux-gnu - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifacts - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Push to Docker Hub - run: | - set -e; ver=${{ env.BUILD_VERSION }} - for tag in $(echo $ver latest); do - docker tag defichain-x86_64-pc-linux-gnu:${ver} defi/defichain:${tag} - done - docker push defi/defichain:${ver} - - linux-aarch64: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - env: - TARGET: aarch64-linux-gnu - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - aarch64-linux-gnu - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu - path: ./build/defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz - - win-x64: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - env: - TARGET: x86_64-w64-mingw32 - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - x86_64-w64-mingw32 - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32 - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.tar.gz - - osx-x64: + create-release: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') + strategy: + matrix: + target: [x86_64-pc-linux-gnu, aarch64-linux-gnu, x86_64-w64-mingw32, x86_64-apple-darwin, aarch64-apple-darwin] env: - TARGET: x86_64-apple-darwin + TARGET: ${{matrix.target}} steps: - uses: actions/checkout@v3 @@ -104,61 +30,16 @@ jobs: - name: Build and package run: ./make.sh docker-release - - name: Publish artifact - x86_64-apple-darwin - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz - - osx-aarch64: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - env: - TARGET: aarch64-apple-darwin + - name: Sign package + run: ./make.sh sign-package - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - aarch64-apple-darwin + - name: Publish artifacts uses: actions/upload-artifact@v3 with: - name: defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin - path: ./build/defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz - - create-release: - needs: - - linux-x64 - - linux-aarch64 - - win-x64 - - osx-x64 - - osx-aarch64 - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Cleanup work dir - run: rm -rf * - - - name: Get artifacts - uses: actions/download-artifact@v3 - - - name: zip package for win-x64 - run: | - set -e; ver=${{ env.BUILD_VERSION }} - cd defichain-${ver}-x86_64-w64-mingw32 - tar xzf defichain-${ver}-x86_64-w64-mingw32.tar.gz - zip -r "defichain-${ver}-x86_64-w64-mingw32.zip" defichain-${ver}/ + name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: | + ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} + ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }}.SHA256 - name: Get release id: get_release @@ -166,116 +47,36 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Generate SHA256 checksum - run: | - set -e; ver=${{ env.BUILD_VERSION }} - (cd ./defichain-${ver}-x86_64-pc-linux-gnu - sha256sum ./defichain-${ver}-x86_64-pc-linux-gnu.tar.gz > ./defichain-${ver}-x86_64-pc-linux-gnu.tar.gz.SHA256) - (cd ./defichain-${ver}-aarch64-linux-gnu - sha256sum ./defichain-${ver}-aarch64-linux-gnu.tar.gz > ./defichain-${ver}-aarch64-linux-gnu.tar.gz.SHA256) - (cd ./defichain-${ver}-x86_64-w64-mingw32 - sha256sum ./defichain-${ver}-x86_64-w64-mingw32.zip > ./defichain-${ver}-x86_64-w64-mingw32.zip.SHA256) - (cd ./defichain-${ver}-x86_64-apple-darwin - sha256sum ./defichain-${ver}-x86_64-apple-darwin.tar.gz > ././defichain-${ver}-x86_64-apple-darwin.tar.gz.SHA256) - (cd ./defichain-${ver}-aarch64-apple-darwin - sha256sum ./defichain-${ver}-aarch64-apple-darwin.tar.gz > ././defichain-${ver}-aarch64-apple-darwin.tar.gz.SHA256) - - - name: Upload release asset - linux-x64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu/defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz - asset_content_type: application/gzip - - - name: Upload checksum - linux-x64 + - name: Upload release asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu/defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz.SHA256 - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz.SHA256 - asset_content_type: text/plain - - - name: Upload release asset - linux-aarch64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu/defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz - asset_name: defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz + asset_path: ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} + asset_name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} asset_content_type: application/gzip - - - name: Upload checksum - linux-aarch64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu/defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz.SHA256 - asset_name: defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz.SHA256 - asset_content_type: text/plain - - name: Upload release asset - win-x64 + - name: Upload checksum uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32/defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.zip - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.zip - asset_content_type: application/zip - - - name: Upload checksum asset - win-x64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32/defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.zip.SHA256 - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.zip.SHA256 + asset_path: ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }}.SHA256 + asset_name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }}.SHA256 asset_content_type: text/plain - - name: Upload release asset - osx-x64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin/defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz - asset_content_type: application/gzip - - - name: Upload checksum asset - osx-x64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin/defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz.SHA256 - asset_name: defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz.SHA256 - asset_content_type: text/plain - - - name: Upload release asset - osx-aarch64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + if: ${{ env.TARGET == 'x86_64-pc-linux-gnu' }} + uses: docker/login-action@v2 with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin/defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz - asset_name: defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz - asset_content_type: application/gzip + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: Upload checksum asset - osx-aarch64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin/defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz.SHA256 - asset_name: defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz.SHA256 - asset_content_type: text/plain + - name: Push to Docker Hub + if: ${{ env.TARGET == 'x86_64-pc-linux-gnu' }} + run: | + set -e; ver=${{ env.BUILD_VERSION }} + docker tag defichain-${{ env.TARGET }}:${ver} defi/defichain:${ver} + docker push defi/defichain:${ver} diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index d04e4d3994..9b43890acb 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -2,16 +2,22 @@ name: Build - Staging on: workflow_dispatch: + push: + branches: + - master env: MAKE_DEBUG: 0 GIT_VERSION: 1 jobs: - linux-x64: + create-release: runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64-pc-linux-gnu, aarch64-linux-gnu, x86_64-w64-mingw32, x86_64-apple-darwin, aarch64-apple-darwin] env: - TARGET: x86_64-pc-linux-gnu + TARGET: ${{matrix.target}} steps: - uses: actions/checkout@v3 @@ -22,134 +28,13 @@ jobs: - name: Build and package run: ./make.sh docker-release - - name: Publish artifacts - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-pc-linux-gnu.tar.gz - - linux-aarch64: - runs-on: ubuntu-latest - env: - TARGET: aarch64-linux-gnu - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - aarch64-linux-gnu - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu - path: ./build/defichain-${{ env.BUILD_VERSION }}-aarch64-linux-gnu.tar.gz - - win-x64: - runs-on: ubuntu-latest - env: - TARGET: x86_64-w64-mingw32 - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - x86_64-w64-mingw32 - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32 - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-w64-mingw32.tar.gz - - osx-x64: - runs-on: ubuntu-latest - env: - TARGET: x86_64-apple-darwin - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release - - - name: Publish artifact - x86_64-apple-darwin - uses: actions/upload-artifact@v3 - with: - name: defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin - path: ./build/defichain-${{ env.BUILD_VERSION }}-x86_64-apple-darwin.tar.gz - - osx-aarch64: - runs-on: ubuntu-latest - env: - TARGET: aarch64-apple-darwin - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Build and package - run: ./make.sh docker-release + - name: Sign package + run: ./make.sh sign-package - - name: Publish artifact - aarch64-apple-darwin + - name: Publish artifacts uses: actions/upload-artifact@v3 with: - name: defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin - path: ./build/defichain-${{ env.BUILD_VERSION }}-aarch64-apple-darwin.tar.gz - - # We keep the flow as similar as close to release flow as possible - # to validate change before it gets into release, even though - # these don't create end artifacts to use - - stage-release: - needs: - - linux-x64 - - linux-aarch64 - - win-x64 - - osx-x64 - - osx-aarch64 - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Populate environment - run: ./make.sh ci-export-vars - - - name: Cleanup work dir - run: rm -rf * - - - name: Get artifacts - uses: actions/download-artifact@v3 - - - name: zip package for win-x64 - run: | - set -e; ver=${{ env.BUILD_VERSION }} - cd defichain-${ver}-x86_64-w64-mingw32 - tar xzf defichain-${ver}-x86_64-w64-mingw32.tar.gz - zip -r "defichain-${ver}-x86_64-w64-mingw32.zip" defichain-${ver}/ - - - name: Generate SHA256 checksum - run: | - set -e; ver=${{ env.BUILD_VERSION }} - (cd ./defichain-${ver}-x86_64-pc-linux-gnu - sha256sum ./defichain-${ver}-x86_64-pc-linux-gnu.tar.gz > ./defichain-${ver}-x86_64-pc-linux-gnu.tar.gz.SHA256) - (cd ./defichain-${ver}-aarch64-linux-gnu - sha256sum ./defichain-${ver}-aarch64-linux-gnu.tar.gz > ./defichain-${ver}-aarch64-linux-gnu.tar.gz.SHA256) - (cd ./defichain-${ver}-x86_64-w64-mingw32 - sha256sum ./defichain-${ver}-x86_64-w64-mingw32.zip > ./defichain-${ver}-x86_64-w64-mingw32.zip.SHA256) - (cd ./defichain-${ver}-x86_64-apple-darwin - sha256sum ./defichain-${ver}-x86_64-apple-darwin.tar.gz > ././defichain-${ver}-x86_64-apple-darwin.tar.gz.SHA256) - (cd ./defichain-${ver}-aarch64-apple-darwin - sha256sum ./defichain-${ver}-aarch64-apple-darwin.tar.gz > ././defichain-${ver}-aarch64-apple-darwin.tar.gz.SHA256) + name: defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }} + path: | + ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }} + ./build/defichain-${{ env.BUILD_VERSION }}-${{ env.TARGET }}.${{ env.PKG_TYPE }}.SHA256 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2709b45b60..5be2269d83 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -41,6 +41,7 @@ jobs: name: Rust Lints runs-on: ubuntu-latest container: defi/ain-builder:latest + steps: - name: Checkout uses: actions/checkout@v3 @@ -58,6 +59,34 @@ jobs: - name: Setup dependencies for test run: ./make.sh ci-setup-deps-test + - name: Restore cached depends build + id: cache-build-deps + uses: actions/cache/restore@v3 + with: + path: ./build/depends + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends + + - name: Restore cached rust build + id: cache-build-rust + uses: actions/cache/restore@v3 + with: + path: ./build/lib + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust + + - name: Restore cached cpp build + id: cache-build-cpp + uses: actions/cache/restore@v3 + with: + path: ./build/src + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp + + - name: Restore cached ccache + id: cache-ccache + uses: actions/cache/restore@v3 + with: + path: ./build/.ccache + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache + - name: Build depends and configure run: ./make.sh build-deps && ./make.sh build-conf diff --git a/.github/workflows/tests-ethlibs.yaml b/.github/workflows/tests-ethlibs.yaml index 65219be52e..0a6c4fc6cc 100644 --- a/.github/workflows/tests-ethlibs.yaml +++ b/.github/workflows/tests-ethlibs.yaml @@ -15,13 +15,17 @@ concurrency: env: NODE_URL: "http://127.0.0.1:19551/" + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 jobs: build: runs-on: ubuntu-latest container: defi/ain-builder:latest + steps: - uses: actions/checkout@v3 + - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars @@ -32,9 +36,37 @@ jobs: - name: Setup dependencies for target run: ./make.sh ci-setup-deps-target - - name: Install dependencies for test + - name: Setup dependencies for test run: ./make.sh ci-setup-deps-test + - name: Cache depends build + id: cache-build-deps + uses: actions/cache@v3 + with: + path: ./build/depends + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends + + - name: Cache rust build + id: cache-build-rust + uses: actions/cache@v3 + with: + path: ./build/lib + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust + + - name: Cache cpp build + id: cache-build-cpp + uses: actions/cache@v3 + with: + path: ./build/src + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp + + - name: Cache ccache + id: cache-ccache + uses: actions/cache@v3 + with: + path: ./build/.ccache + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache + - name: Build binaries run: ./make.sh build diff --git a/.github/workflows/tests-frontier.yml b/.github/workflows/tests-frontier.yml index 86566e1573..e6236994fb 100644 --- a/.github/workflows/tests-frontier.yml +++ b/.github/workflows/tests-frontier.yml @@ -13,24 +13,59 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} +env: + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + jobs: build: runs-on: ubuntu-latest + container: defi/ain-builder:latest + steps: - uses: actions/checkout@v3 + - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars - name: Setup dependencies - run: sudo ./make.sh ci-setup-deps + run: ./make.sh ci-setup-deps - name: Setup dependencies for target run: ./make.sh ci-setup-deps-target - - name: Install dependencies for test + - name: Setup dependencies for test run: ./make.sh ci-setup-deps-test + - name: Restore cached depends build + id: cache-build-deps + uses: actions/cache/restore@v3 + with: + path: ./build/depends + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends + + - name: Restore cached rust build + id: cache-build-rust + uses: actions/cache/restore@v3 + with: + path: ./build/lib + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust + + - name: Restore cached cpp build + id: cache-build-cpp + uses: actions/cache/restore@v3 + with: + path: ./build/src + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp + + - name: Restore cached ccache + id: cache-ccache + uses: actions/cache/restore@v3 + with: + path: ./build/.ccache + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache + - name: Build binaries run: ./make.sh build diff --git a/.github/workflows/tests-jellyfish.yml b/.github/workflows/tests-jellyfish.yml index 9289df17df..b3f91a8694 100644 --- a/.github/workflows/tests-jellyfish.yml +++ b/.github/workflows/tests-jellyfish.yml @@ -12,6 +12,9 @@ concurrency: env: BUILD_VERSION: latest # Computed + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + GIT_VERSION: 1 jobs: test: @@ -32,11 +35,12 @@ jobs: node-version: '18' - name: Populate environment - run: cd defichain && GIT_VERSION=1 ./make.sh ci-export-vars + run: cd defichain && ./make.sh ci-export-vars - name: Build and setup - run: "cd defichain && docker build -t test-build-container - -f ./contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile ." + run: | + cd defichain && ./make.sh docker-build + docker tag defichain-${{ env.TARGET }}:${{ env.BUILD_VERSION }} defichain-test-build-container - name: Run tests run: | diff --git a/.github/workflows/tests-sync.yml b/.github/workflows/tests-sync.yml index 72dbe54401..b9dd167ad4 100644 --- a/.github/workflows/tests-sync.yml +++ b/.github/workflows/tests-sync.yml @@ -25,6 +25,7 @@ jobs: build: if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' runs-on: [self-hosted, linux, x64, builder] + steps: - uses: actions/checkout@v3 @@ -37,11 +38,9 @@ jobs: - name: Setup dependencies for target run: ./make.sh ci-setup-deps-target - - name: Install dependencies for test + - name: Setup dependencies for test run: sudo ./make.sh ci-setup-deps-test - # TODO: Switch this to a docker build later and this builds on the native - # VM toolchain and loses disparity with the CI release builds - name: Build binaries run: ./make.sh build diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 344b3bca98..c51ab4aa79 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -14,37 +14,71 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} - + +env: + TARGET: x86_64-pc-linux-gnu + MAKE_DEBUG: 0 + jobs: e2e-tests: runs-on: ubuntu-latest container: defi/ain-builder:latest env: GITHUB_PULL_REQUEST: ${{ github.event.number }} + steps: - name: Checkout base branch and/or merge if: github.event_name != 'pull_request' - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - run: git config --global --add safe.directory '*' - name: Checkout pull request head commit if: github.event_name == 'pull_request' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Populate environment run: ./make.sh ci-export-vars - - name: Install dependencies + - name: Setup dependencies run: ./make.sh ci-setup-deps - - name: Install dependencies for target + - name: Setup dependencies for target run: ./make.sh ci-setup-deps-target - - name: Install dependencies for test + - name: Setup dependencies for test run: ./make.sh ci-setup-deps-test - - name: Build deps and configure + - name: Restore cached depends build + id: cache-build-deps + uses: actions/cache/restore@v3 + with: + path: ./build/depends + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends + + - name: Restore cached rust build + id: cache-build-rust + uses: actions/cache/restore@v3 + with: + path: ./build/lib + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust + + - name: Restore cached cpp build + id: cache-build-cpp + uses: actions/cache/restore@v3 + with: + path: ./build/src + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp + + - name: Restore cached ccache + id: cache-ccache + uses: actions/cache/restore@v3 + with: + path: ./build/.ccache + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache + + - name: Build binaries run: ./make.sh build - name: E2E tests @@ -53,9 +87,11 @@ jobs: rust-tests: runs-on: ubuntu-latest container: defi/ain-builder:latest + steps: - name: Checkout uses: actions/checkout@v3 + - run: git config --global --add safe.directory '*' - name: Populate environment run: ./make.sh ci-export-vars @@ -66,9 +102,37 @@ jobs: - name: Setup dependencies for target run: ./make.sh ci-setup-deps-target - - name: Install dependencies for test + - name: Setup dependencies for test run: ./make.sh ci-setup-deps-test + - name: Restore cached depends build + id: cache-build-deps + uses: actions/cache/restore@v3 + with: + path: ./build/depends + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-depends + + - name: Restore cached rust build + id: cache-build-rust + uses: actions/cache/restore@v3 + with: + path: ./build/lib + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-rust + + - name: Restore cached cpp build + id: cache-build-cpp + uses: actions/cache/restore@v3 + with: + path: ./build/src + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cpp + + - name: Restore cached ccache + id: cache-ccache + uses: actions/cache/restore@v3 + with: + path: ./build/.ccache + key: ${{ env.TARGET }}-${{ env.BUILD_TYPE }}-cache + - name: Build deps and configure run: ./make.sh build-deps && ./make.sh build-conf diff --git a/.gitignore b/.gitignore index 33d619c7e4..2f433d34e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .code .cache +.ccache .idea # Editors. diff --git a/contrib/dockerfiles/noarch.dockerfile b/contrib/dockerfiles/noarch.dockerfile index b305795642..3d8a40454e 100644 --- a/contrib/dockerfiles/noarch.dockerfile +++ b/contrib/dockerfiles/noarch.dockerfile @@ -7,6 +7,7 @@ ARG TARGET=unknown FROM ubuntu:latest as builder ARG TARGET ARG MAKE_DEBUG +ARG CCACHE_DIR="/work/build/.ccache" LABEL org.defichain.name="defichain-builder" LABEL org.defichain.arch=${TARGET} diff --git a/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile b/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile index 8f0a04e440..b5069c4566 100644 --- a/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile +++ b/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile @@ -5,6 +5,7 @@ ARG TARGET=x86_64-pc-linux-gnu FROM --platform=linux/amd64 docker.io/defi/ain-builder as builder ARG TARGET ARG MAKE_DEBUG +ARG CCACHE_DIR="/work/build/.ccache" LABEL org.defichain.name="defichain-builder" LABEL org.defichain.arch=${TARGET} @@ -26,7 +27,7 @@ RUN mkdir /app && cd build/ && \ # ----------- ### Actual image that contains defi binaries -FROM --platform=linux/amd64 ubuntu:latest +FROM --platform=linux/amd64 ubuntu:latest as defi ARG TARGET ENV PATH=/app/bin:$PATH LABEL org.defichain.name="defichain" diff --git a/make.sh b/make.sh index 42eed33ac7..49c4791ca0 100755 --- a/make.sh +++ b/make.sh @@ -19,10 +19,12 @@ setup_vars() { DOCKER_ROOT_CONTEXT=${DOCKER_ROOT_CONTEXT:-"."} DOCKERFILES_DIR=${DOCKERFILES_DIR:-"./contrib/dockerfiles"} + DOCKER_TARGET=${DOCKER_TARGET:-"builder"} ROOT_DIR="$(_canonicalize "${_SCRIPT_DIR}")" TARGET=${TARGET:-"$(get_default_target)"} + PACKAGE=${PACKAGE:-"$(get_default_package)"} DOCKERFILE=${DOCKERFILE:-"$(get_default_docker_file)"} BUILD_DIR=${BUILD_DIR:-"./build"} @@ -186,6 +188,8 @@ build() { build_make "$@" } +# ----------- Deployment --------------- + deploy() { local target=${1:-${TARGET}} local img_prefix="${IMAGE_PREFIX}" @@ -212,12 +216,7 @@ package() { local img_prefix="${IMAGE_PREFIX}" local img_version="${IMAGE_VERSION}" local build_dir="${BUILD_DIR}" - - local pkg_name="${img_prefix}-${img_version}-${target}" - local pkg_tar_file_name="${pkg_name}.tar.gz" - - local pkg_path - pkg_path="$(_canonicalize "${build_dir}/${pkg_tar_file_name}")" + local pkg_name="${PACKAGE}" local versioned_name="${img_prefix}-${img_version}" local versioned_build_dir="${build_dir}/${versioned_name}" @@ -230,26 +229,56 @@ package() { echo "> packaging: ${pkg_name} from ${versioned_build_dir}" - _ensure_enter_dir "${versioned_build_dir}" - _tar --transform "s,^./,${versioned_name}/," -czf "${pkg_path}" ./* - _exit_dir + local pkg_path + if [[ "$target" == "x86_64-w64-mingw32" ]]; then + local pkg_zip_file_name="${pkg_name}.zip" + pkg_path="$(_canonicalize "${build_dir}/${pkg_zip_file_name}")" + + _ensure_enter_dir "${build_dir}" + zip -r "${pkg_zip_file_name}" "${versioned_name}/" + _exit_dir + else + local pkg_tar_file_name="${pkg_name}.tar.gz" + pkg_path="$(_canonicalize "${build_dir}/${pkg_tar_file_name}")" + + _ensure_enter_dir "${versioned_build_dir}" + _tar --transform "s,^./,${versioned_name}/," -czf "${pkg_path}" ./* + _exit_dir + fi echo "> package: ${pkg_path}" } +sign_package() { + local pkg_name="${PACKAGE}" + local pkg_file_name="${pkg_name}.tar.gz" + local pkg_sign_file_name="${pkg_name}.tar.gz.SHA256" + local build_dir="${BUILD_DIR}" + + if [[ "$target" == "x86_64-w64-mingw32" ]]; then + pkg_file_name="${pkg_name}.zip" + pkg_sign_file_name="${pkg_name}.zip.SHA256" + fi + + local pkg_path + pkg_path="$(_canonicalize "${build_dir}/${pkg_file_name}")" + pkg_sign_path="$(_canonicalize "${build_dir}/${pkg_sign_file_name}")" + sha256sum "$pkg_path" > "$pkg_sign_path" +} + release() { local target=${1:-${TARGET}} build "${target}" deploy "${target}" package "${target}" - _sign "${target}" } # -------------- Docker --------------- docker_build() { local target=${1:-${TARGET}} + local stage="${DOCKER_TARGET}" local img_prefix="${IMAGE_PREFIX}" local img_version="${IMAGE_VERSION}" local docker_context="${DOCKER_ROOT_CONTEXT}" @@ -257,12 +286,13 @@ docker_build() { echo "> docker-build"; - local img="${img_prefix}-${target}:${img_version}" + local img=${IMAGE_NAME:-"${img_prefix}-${target}:${img_version}"} echo "> building: ${img}" echo "> docker build: ${img}" docker build -f "${docker_file}" \ --build-arg TARGET="${target}" \ --build-arg MAKE_DEBUG="${MAKE_DEBUG}" \ + --target "${stage}" \ -t "${img}" "${docker_context}" } @@ -277,7 +307,6 @@ docker_deploy() { local img="${img_prefix}-${target}:${img_version}" echo "> deploy from: ${img}" - local pkg_name="${img_prefix}-${img_version}-${target}" local versioned_name="${img_prefix}-${img_version}" local versioned_build_dir="${build_dir}/${versioned_name}" @@ -297,13 +326,44 @@ docker_deploy() { fi } +docker_deploy_build() { + local target=${1:-${TARGET}} + local img_prefix="${IMAGE_PREFIX}" + local img_version="${IMAGE_VERSION}" + local build_dir="${BUILD_DIR}" + + echo "> docker-deploy-build"; + + local img="${img_prefix}-${target}:${img_version}" + echo "> deploy from: ${img}" + + local pkg_name="${img_prefix}-${img_version}-${target}" + local versioned_name="${img_prefix}-${img_version}" + + local cid + cid=$(docker create "${img}") + local e=0 + + # TODO: grab build directories from builder image instead of defi image + { docker cp "${cid}:/work/build/depends" "${build_dir}/depends" 2>/dev/null && e=1; } || true + { docker cp "${cid}:/work/build/lib" "${build_dir}/lib" 2>/dev/null && e=1; } || true + { docker cp "${cid}:/work/build/src" "${build_dir}/src" 2>/dev/null && e=1; } || true + { docker cp "${cid}:/work/build/.ccache" "${build_dir}/.ccache" 2>/dev/null && e=1; } || true + docker rm "${cid}" + + if [[ "$e" == "1" ]]; then + echo "> deployed into: ${build_dir}" + else + echo "> failed: please ensure defi is built first" + fi +} + docker_release() { local target=${1:-${TARGET}} docker_build "$target" docker_deploy "$target" package "$target" - _sign "$target" } docker_clean_builds() { @@ -450,7 +510,7 @@ pkg_install_deps() { software-properties-common build-essential git libtool autotools-dev automake \ pkg-config bsdmainutils python3 python3-pip libssl-dev libevent-dev libboost-system-dev \ libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev \ - libminiupnpc-dev libzmq3-dev libqrencode-dev wget \ + libminiupnpc-dev libzmq3-dev libqrencode-dev wget ccache \ libdb-dev libdb++-dev libdb5.3 libdb5.3-dev libdb5.3++ libdb5.3++-dev \ curl cmake unzip libc6-dev gcc-multilib locales locales-all @@ -694,6 +754,14 @@ get_default_target() { echo "$default_target" } +get_default_package() { + local target="${TARGET}" + local img_prefix="${IMAGE_PREFIX}" + local img_version="${IMAGE_VERSION}" + local pkg_name="${img_prefix}-${img_version}-${target}" + echo "$pkg_name" +} + get_default_docker_file() { local target="${TARGET}" local dockerfiles_dir="${DOCKERFILES_DIR}" @@ -894,10 +962,27 @@ _nproc() { # --- ci_export_vars() { + local build_dir="${BUILD_DIR}" + + # GitHub Actions if [[ -n "${GITHUB_ACTIONS-}" ]]; then - # GitHub Actions - echo "BUILD_VERSION=${IMAGE_VERSION}" >> "$GITHUB_ENV" - echo "PATH=$HOME/.cargo/bin:$PATH" >> "$GITHUB_ENV" + { + echo "BUILD_VERSION=${IMAGE_VERSION}" + echo "PATH=$HOME/.cargo/bin:$PATH" + echo "CCACHE_DIR=${build_dir}/.ccache" + } >> "$GITHUB_ENV" + + if [[ "${MAKE_DEBUG}" == "1" ]]; then + echo "BUILD_TYPE=debug" >> "$GITHUB_ENV" + else + echo "BUILD_TYPE=release" >> "$GITHUB_ENV" + fi + + if [[ "${TARGET}" == "x86_64-w64-mingw32" ]]; then + echo "PKG_TYPE=zip" >> "$GITHUB_ENV" + else + echo "PKG_TYPE=tar.gz" >> "$GITHUB_ENV" + fi fi } @@ -958,11 +1043,6 @@ get_rust_target() { echo "$rust_target" } -_sign() { - # TODO: generate sha sums and sign - : -} - _safe_rm_rf() { local x for x in "$@"; do