From 32a0a373c491578c00e7f1df40e03af106b3bc2c Mon Sep 17 00:00:00 2001 From: "C.Lee Taylor" Date: Tue, 8 Oct 2024 11:26:40 +0200 Subject: [PATCH] chore(ci): add riscv64 builds and misc script fixes --- .github/workflows/build_binaries.json | 11 ++- .github/workflows/build_binaries.yml | 1 + Cargo.lock | 41 +++-------- Cross.toml | 18 +++++ applications/minotari_app_grpc/Cargo.toml | 2 +- .../minotari_app_grpc/src/tls/error.rs | 4 +- .../minotari_app_utilities/Cargo.toml | 1 - scripts/cross_compile_ubuntu_18-pre-build.sh | 73 ++++++++++++++----- scripts/file_license_check.sh | 68 +++++++++++++++-- scripts/install_ubuntu_dependencies.sh | 1 + 10 files changed, 154 insertions(+), 66 deletions(-) diff --git a/.github/workflows/build_binaries.json b/.github/workflows/build_binaries.json index dd7466999b..c1210b618f 100644 --- a/.github/workflows/build_binaries.json +++ b/.github/workflows/build_binaries.json @@ -13,7 +13,7 @@ "rust": "stable", "target": "aarch64-unknown-linux-gnu", "cross": true, - "flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", + "flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", "build_metric": true }, { @@ -22,13 +22,15 @@ "rust": "stable", "target": "riscv64gc-unknown-linux-gnu", "cross": true, - "flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", + "features": "safe", + "flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests", + "build_metric": true, "build_enabled": true, "best_effort": true }, { "name": "macos-x86_64", - "runs-on": "macos-12", + "runs-on": "macos-13", "rust": "stable", "target": "x86_64-apple-darwin", "cross": false @@ -58,6 +60,7 @@ "features": "safe", "target_bins": "minotari_node, minotari_console_wallet, minotari_merge_mining_proxy, minotari_miner", "flags": "--workspace --exclude tari_libtor", - "build_enabled": false + "build_enabled": true, + "best_effort": true } ] diff --git a/.github/workflows/build_binaries.yml b/.github/workflows/build_binaries.yml index b3c9b11243..12ab8c1db3 100644 --- a/.github/workflows/build_binaries.yml +++ b/.github/workflows/build_binaries.yml @@ -227,6 +227,7 @@ jobs: # Should already be installed # choco upgrade strawberryperl -y choco upgrade protoc -y + rustup target add ${{ matrix.builds.target }} - name: Set environment variables - Nix if: ${{ ! startsWith(runner.os,'Windows') }} diff --git a/Cargo.lock b/Cargo.lock index e142596d0c..9be216f546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2583,7 +2583,7 @@ dependencies = [ "ipnet", "once_cell", "rand", - "ring 0.17.8", + "ring", "rustls", "rustls-pemfile 2.2.0", "thiserror", @@ -5243,9 +5243,9 @@ dependencies = [ [[package]] name = "randomx-rs" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14fb999f322669968fd0e80aeca5cb91e7a817a94ebf2b0fcd345a4a7c695203" +checksum = "9abb8f2aa3432700c2b64a67406ac0da4956d78991f50559509cecc2b6abf249" dependencies = [ "bitflags 1.3.2", "libc", @@ -5274,12 +5274,12 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.11.3" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" +checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" dependencies = [ "pem", - "ring 0.16.20", + "ring", "time", "yasna", ] @@ -5421,21 +5421,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.8" @@ -5447,7 +5432,7 @@ dependencies = [ "getrandom", "libc", "spin 0.9.8", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -5554,7 +5539,7 @@ checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", - "ring 0.17.8", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -5604,9 +5589,9 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ - "ring 0.17.8", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -7708,12 +7693,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cross.toml b/Cross.toml index ab4927164e..a10f67a000 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,6 +7,7 @@ passthrough = [ "CARGO_BUILD_TARGET", "TARGET_CFLAGS", "CC_aarch64_unknown_linux_gnu", + "CC_riscv64_unknown_linux_gnu", "PKG_CONFIG_SYSROOT_DIR", "PKG_CONFIG_ALLOW_CROSS", "RUSTFLAGS", @@ -72,3 +73,20 @@ passthrough = [ [target.x86_64-unknown-linux-gnu] image = "ubuntu:18.04" pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh" + +[target.x86_64-unknown-linux-gnu.env] +passthrough = [ + "PKG_CONFIG_ALLOW_CROSS=true", +] + +[target.riscv64gc-unknown-linux-gnu] +#image.name = "riscv64/ubuntu:22.04" +#image.toolchain = ["linux/riscv64=riscv64-unknown-linux-gnu", "linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"] +image.name = "ubuntu:22.04" +image.toolchain = ["linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu", "linux/riscv64=riscv64gc-unknown-linux-gnu"] +pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh" + +[target.riscv64gc-unknown-linux-gnu.env] +passthrough = [ + "PKG_CONFIG_ALLOW_CROSS=true", +] diff --git a/applications/minotari_app_grpc/Cargo.toml b/applications/minotari_app_grpc/Cargo.toml index 076d4241df..eaf23a5d8b 100644 --- a/applications/minotari_app_grpc/Cargo.toml +++ b/applications/minotari_app_grpc/Cargo.toml @@ -24,7 +24,7 @@ log = "0.4" prost = "0.13.3" prost-types = "0.13.3" rand = "0.8" -rcgen = "0.11.3" +rcgen = "0.12.1" subtle = "2.5.0" thiserror = "1" tokio = { version = "1.36", features = ["fs"] } diff --git a/applications/minotari_app_grpc/src/tls/error.rs b/applications/minotari_app_grpc/src/tls/error.rs index 2f843138c6..a7da07346d 100644 --- a/applications/minotari_app_grpc/src/tls/error.rs +++ b/applications/minotari_app_grpc/src/tls/error.rs @@ -22,14 +22,12 @@ use std::io; -use rcgen::RcgenError; - #[derive(Debug, thiserror::Error)] pub enum GrpcTlsError { #[error("Couldn't read file: {0}")] FileReadError(String), #[error("Error generating the certificate: {0}")] - CertGenerationError(#[from] RcgenError), + CertGenerationError(#[from] rcgen::Error), #[error("Error opening or writing the file: {0}")] IoError(#[from] io::Error), } diff --git a/applications/minotari_app_utilities/Cargo.toml b/applications/minotari_app_utilities/Cargo.toml index 3240758d25..8b05ee17ca 100644 --- a/applications/minotari_app_utilities/Cargo.toml +++ b/applications/minotari_app_utilities/Cargo.toml @@ -25,7 +25,6 @@ thiserror = "^1.0.26" dialoguer = { version = "0.10" } tonic = "0.12.3" - [build-dependencies] tari_common = { path = "../../common", features = [ "build", diff --git a/scripts/cross_compile_ubuntu_18-pre-build.sh b/scripts/cross_compile_ubuntu_18-pre-build.sh index e93f2a10fc..2292bd198f 100755 --- a/scripts/cross_compile_ubuntu_18-pre-build.sh +++ b/scripts/cross_compile_ubuntu_18-pre-build.sh @@ -5,40 +5,69 @@ set -e -USAGE="Usage: $0 target build ie: x86_64-unknown-linux-gnu or aarch64-unknown-linux-gnu" +# APT Proxy for quicker testing +#export HTTP_PROXY_APT=http://apt-proxy.local:3142 +if [ ! -z "${HTTP_PROXY_APT}" ] && [ -d "/etc/apt/apt.conf.d/" ]; then + echo "Setup apt proxy - ${HTTP_PROXY_APT}" + cat << APT-EoF > /etc/apt/apt.conf.d/proxy.conf +Acquire { + HTTP::proxy "${HTTP_PROXY_APT}"; + #HTTPS::proxy "http://127.0.0.1:8080"; +} +APT-EoF +fi + +USAGE="Usage: $0 {target build} + where target build is one of the following: + x86_64-unknown-linux-gnu or + aarch64-unknown-linux-gnu or + riscv64gc-unknown-linux-gnu +" if [ "$#" == "0" ]; then - echo "$USAGE" + echo -e "${USAGE}" exit 1 fi if [ -z "${CROSS_DEB_ARCH}" ]; then - echo "Should be run from cross, which sets the env CROSS_DEB_ARCH" + echo -e "Should be run from cross, which sets the env 'CROSS_DEB_ARCH' + amd64 + arm64 + riscv64 +" exit 1 fi +DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive} +# Hack of Note! +TimeZone=${TimeZone:-"Etc/GMT"} +ln -snf /usr/share/zoneinfo/${TimeZone} /etc/localtime +echo ${TimeZone} > /etc/timezone + targetBuild="${1}" nativeRunTime=$(uname -m) echo "Native RunTime is ${nativeRunTime}" if [ "${nativeRunTime}" == "x86_64" ]; then nativeArch=amd64 - if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then - targetArch=arm64 - targetPlatform=aarch64 - else - targetArch=amd64 - targetPlatform=x86-64 - fi elif [ "${nativeRunTime}" == "aarch64" ]; then nativeArch=arm64 - if [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then - targetArch=amd64 - targetPlatform=x86-64 - fi elif [ "${nativeRunTime}" == "riscv64" ]; then nativeArch=riscv64 - echo "ToDo!" +else + echo "!!Unsupport platform!!" + exit 1 +fi + +if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then + targetArch=arm64 + targetPlatform=aarch64 +elif [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then + targetArch=amd64 + targetPlatform=x86-64 +elif [ "${targetBuild}" == "riscv64gc-unknown-linux-gnu" ]; then + targetArch=riscv64 + targetPlatform=riscv64 else echo "!!Unsupport platform!!" exit 1 @@ -63,6 +92,7 @@ apt-get install --no-install-recommends --assume-yes \ libsqlite3-0 \ libreadline-dev \ git \ + make \ cmake \ dh-autoreconf \ clang \ @@ -91,7 +121,7 @@ if [ "${CROSS_DEB_ARCH}" != "${nativeArch}" ]; then . /etc/lsb-release ubuntu_tag=${DISTRIB_CODENAME} - if [ "${crossArch}" == "arm64" ]; then + if [[ "${crossArch}" =~ ^(arm|riscv)64$ ]]; then cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse # deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse @@ -143,24 +173,31 @@ deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multi EoF fi + dpkg --print-architecture dpkg --add-architecture ${CROSS_DEB_ARCH} + dpkg --print-architecture apt-get update # scripts/install_ubuntu_dependencies-cross_compile.sh x86-64 +# pkg-config-${targetPlatform}-linux-gnu \ apt-get --assume-yes install \ - pkg-config-${targetPlatform}-linux-gnu \ gcc-${targetPlatform}-linux-gnu \ g++-${targetPlatform}-linux-gnu # packages needed for Ledger and hidapi apt-get --assume-yes install \ libudev-dev:${CROSS_DEB_ARCH} \ - libhidapi-dev:${CROSS_DEB_ARCH} + libhidapi-dev:${CROSS_DEB_ARCH} \ + libssl-dev:${CROSS_DEB_ARCH} fi +rustup show + rustup target add ${targetBuild} rustup toolchain install stable-${targetBuild} --force-non-host rustup target list --installed rustup toolchain list + +rustup show diff --git a/scripts/file_license_check.sh b/scripts/file_license_check.sh index b33d814570..ce1feb1c75 100755 --- a/scripts/file_license_check.sh +++ b/scripts/file_license_check.sh @@ -1,5 +1,49 @@ -#!/bin/bash +#!/usr/bin/env bash +# # Must be run from the repo root +# + +#set -xo pipefail +set -e + +check_for() { + if prog_location=$(which ${1}) ; then + if result=$(${1} --version 2>/dev/null); then + result="${1}: ${result} INSTALLED ✓" + else + result="${1}: INSTALLED ✓" + fi + else + result="${1}: MISSING ⨯" + fi +} + +check_requirements() { + echo "List of requirements and possible test:" + req_progs=( + mktemp + rg + diff + ) + for RProg in "${req_progs[@]}"; do + check_for ${RProg} + echo "${result}" + if [[ "${result}" == "${RProg}: MISSING ⨯" ]]; then + echo "!! Install ${RProg} and try again !!" + exit -1 + fi + done + + if [ ! -f .license.ignore ]; then + echo "!! No .license.ignore file !!" + exit -1 + fi +} + +check_requirements + +diffparms=${diffparms:-"-u --suppress-blank-empty --strip-trailing-cr --color=never"} +rgTemp=${rgTemp:-$(mktemp)} # rg -i "Copyright.*The Tari Project" --files-without-match \ # -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \ @@ -8,23 +52,31 @@ # Exclude files without extensions as well as those with extensions that are not in the list # rg -i "Copyright.*The Tari Project" --files-without-match \ - -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \ + -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue,liquid,otf,d.ts,mjs}' . \ | while IFS= read -r file; do if [[ -n $(basename "$file" | grep -E '\.') ]]; then echo "$file" fi - done | sort > /tmp/rgtemp + done | sort > ${rgTemp} # Sort the .license.ignore file as sorting seems to behave differently on different platforms -cat .license.ignore | sort > /tmp/.license.ignore +licenseIgnoreTemp=${licenseIgnoreTemp:-$(mktemp)} +cat .license.ignore | sort > ${licenseIgnoreTemp} + +DIFFS=$( diff ${diffparms} ${licenseIgnoreTemp} ${rgTemp} || true ) -DIFFS=$(diff -u --strip-trailing-cr /tmp/.license.ignore /tmp/rgtemp) +# clean up +rm -vf ${rgTemp} +rm -vf ${licenseIgnoreTemp} -if [ -n "$DIFFS" ]; then - echo "New files detected that either need copyright/license identifiers added, or they need to be added to .license.ignore" +if [ -n "${DIFFS}" ]; then + echo "New files detected that either need copyright/license identifiers added, " + echo "or they need to be added to .license.ignore" echo "NB: The ignore file must be sorted alphabetically!" echo "Diff:" - echo "$DIFFS" + echo "${DIFFS}" exit 1 +else + exit 0 fi diff --git a/scripts/install_ubuntu_dependencies.sh b/scripts/install_ubuntu_dependencies.sh index b326d2207e..375b4e6166 100755 --- a/scripts/install_ubuntu_dependencies.sh +++ b/scripts/install_ubuntu_dependencies.sh @@ -12,6 +12,7 @@ apt-get install --no-install-recommends --assume-yes \ libsqlite3-0 \ libreadline-dev \ git \ + make \ cmake \ dh-autoreconf \ clang \