From 4ee8ae824503c6c0d1cadee52b027ea6a06c485d Mon Sep 17 00:00:00 2001
From: "C.Lee Taylor" <47312074+leet4tari@users.noreply.github.com>
Date: Tue, 8 Oct 2024 11:59:52 +0200
Subject: [PATCH] chore(ci): add riscv64 builds and misc script fixes (#6611)

Description
Bump rcgen crate (removes ring v0.16)
Extend cross-compile scripts to support riscv64
Add make to build system, ubuntu 20.04 and above don't include by
default.

Motivation and Context
Add working riscv64 builds - enabled by default

How Has This Been Tested?
Builds locally, both emulated and cross-compile for riscv64
---
 .github/workflows/build_binaries.json         | 11 ++-
 .github/workflows/build_binaries.yml          |  1 +
 Cargo.lock                                    | 41 +++--------
 Cross.toml                                    | 18 +++++
 applications/minotari_app_grpc/Cargo.toml     |  2 +-
 .../minotari_app_grpc/src/tls/error.rs        |  4 +-
 .../minotari_app_utilities/Cargo.toml         |  1 -
 scripts/cross_compile_ubuntu_18-pre-build.sh  | 73 ++++++++++++++-----
 scripts/file_license_check.sh                 | 68 +++++++++++++++--
 scripts/install_ubuntu_dependencies.sh        |  1 +
 10 files changed, 154 insertions(+), 66 deletions(-)

diff --git a/.github/workflows/build_binaries.json b/.github/workflows/build_binaries.json
index dd7466999b..c1210b618f 100644
--- a/.github/workflows/build_binaries.json
+++ b/.github/workflows/build_binaries.json
@@ -13,7 +13,7 @@
     "rust": "stable",
     "target": "aarch64-unknown-linux-gnu",
     "cross": true,
-    "flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
+    "flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
     "build_metric": true
   },
   {
@@ -22,13 +22,15 @@
     "rust": "stable",
     "target": "riscv64gc-unknown-linux-gnu",
     "cross": true,
-    "flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
+    "features": "safe",
+    "flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
+    "build_metric": true,
     "build_enabled": true,
     "best_effort": true
   },
   {
     "name": "macos-x86_64",
-    "runs-on": "macos-12",
+    "runs-on": "macos-13",
     "rust": "stable",
     "target": "x86_64-apple-darwin",
     "cross": false
@@ -58,6 +60,7 @@
     "features": "safe",
     "target_bins": "minotari_node, minotari_console_wallet, minotari_merge_mining_proxy, minotari_miner",
     "flags": "--workspace --exclude tari_libtor",
-    "build_enabled": false
+    "build_enabled": true,
+    "best_effort": true
   }
 ]
diff --git a/.github/workflows/build_binaries.yml b/.github/workflows/build_binaries.yml
index b3c9b11243..12ab8c1db3 100644
--- a/.github/workflows/build_binaries.yml
+++ b/.github/workflows/build_binaries.yml
@@ -227,6 +227,7 @@ jobs:
           # Should already be installed
           # choco upgrade strawberryperl -y
           choco upgrade protoc -y
+          rustup target add ${{ matrix.builds.target }}
 
       - name: Set environment variables - Nix
         if: ${{ ! startsWith(runner.os,'Windows') }}
diff --git a/Cargo.lock b/Cargo.lock
index e142596d0c..9be216f546 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2583,7 +2583,7 @@ dependencies = [
  "ipnet",
  "once_cell",
  "rand",
- "ring 0.17.8",
+ "ring",
  "rustls",
  "rustls-pemfile 2.2.0",
  "thiserror",
@@ -5243,9 +5243,9 @@ dependencies = [
 
 [[package]]
 name = "randomx-rs"
-version = "1.3.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14fb999f322669968fd0e80aeca5cb91e7a817a94ebf2b0fcd345a4a7c695203"
+checksum = "9abb8f2aa3432700c2b64a67406ac0da4956d78991f50559509cecc2b6abf249"
 dependencies = [
  "bitflags 1.3.2",
  "libc",
@@ -5274,12 +5274,12 @@ dependencies = [
 
 [[package]]
 name = "rcgen"
-version = "0.11.3"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6"
+checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1"
 dependencies = [
  "pem",
- "ring 0.16.20",
+ "ring",
  "time",
  "yasna",
 ]
@@ -5421,21 +5421,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "ring"
-version = "0.16.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
-dependencies = [
- "cc",
- "libc",
- "once_cell",
- "spin 0.5.2",
- "untrusted 0.7.1",
- "web-sys",
- "winapi",
-]
-
 [[package]]
 name = "ring"
 version = "0.17.8"
@@ -5447,7 +5432,7 @@ dependencies = [
  "getrandom",
  "libc",
  "spin 0.9.8",
- "untrusted 0.9.0",
+ "untrusted",
  "windows-sys 0.52.0",
 ]
 
@@ -5554,7 +5539,7 @@ checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8"
 dependencies = [
  "log",
  "once_cell",
- "ring 0.17.8",
+ "ring",
  "rustls-pki-types",
  "rustls-webpki",
  "subtle",
@@ -5604,9 +5589,9 @@ version = "0.102.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
- "ring 0.17.8",
+ "ring",
  "rustls-pki-types",
- "untrusted 0.9.0",
+ "untrusted",
 ]
 
 [[package]]
@@ -7708,12 +7693,6 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105"
 
-[[package]]
-name = "untrusted"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"
diff --git a/Cross.toml b/Cross.toml
index ab4927164e..a10f67a000 100644
--- a/Cross.toml
+++ b/Cross.toml
@@ -7,6 +7,7 @@ passthrough = [
   "CARGO_BUILD_TARGET",
   "TARGET_CFLAGS",
   "CC_aarch64_unknown_linux_gnu",
+  "CC_riscv64_unknown_linux_gnu",
   "PKG_CONFIG_SYSROOT_DIR",
   "PKG_CONFIG_ALLOW_CROSS",
   "RUSTFLAGS",
@@ -72,3 +73,20 @@ passthrough = [
 [target.x86_64-unknown-linux-gnu]
 image = "ubuntu:18.04"
 pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh"
+
+[target.x86_64-unknown-linux-gnu.env]
+passthrough = [
+  "PKG_CONFIG_ALLOW_CROSS=true",
+]
+
+[target.riscv64gc-unknown-linux-gnu]
+#image.name = "riscv64/ubuntu:22.04"
+#image.toolchain = ["linux/riscv64=riscv64-unknown-linux-gnu", "linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu"]
+image.name = "ubuntu:22.04"
+image.toolchain = ["linux/arm64=aarch64-unknown-linux-gnu", "linux/amd64=x86_64-unknown-linux-gnu", "linux/riscv64=riscv64gc-unknown-linux-gnu"]
+pre-build = "./scripts/cross_compile_ubuntu_18-pre-build.sh"
+
+[target.riscv64gc-unknown-linux-gnu.env]
+passthrough = [
+  "PKG_CONFIG_ALLOW_CROSS=true",
+]
diff --git a/applications/minotari_app_grpc/Cargo.toml b/applications/minotari_app_grpc/Cargo.toml
index 076d4241df..eaf23a5d8b 100644
--- a/applications/minotari_app_grpc/Cargo.toml
+++ b/applications/minotari_app_grpc/Cargo.toml
@@ -24,7 +24,7 @@ log = "0.4"
 prost = "0.13.3"
 prost-types = "0.13.3"
 rand = "0.8"
-rcgen = "0.11.3"
+rcgen = "0.12.1"
 subtle = "2.5.0"
 thiserror = "1"
 tokio = { version = "1.36", features = ["fs"] }
diff --git a/applications/minotari_app_grpc/src/tls/error.rs b/applications/minotari_app_grpc/src/tls/error.rs
index 2f843138c6..a7da07346d 100644
--- a/applications/minotari_app_grpc/src/tls/error.rs
+++ b/applications/minotari_app_grpc/src/tls/error.rs
@@ -22,14 +22,12 @@
 
 use std::io;
 
-use rcgen::RcgenError;
-
 #[derive(Debug, thiserror::Error)]
 pub enum GrpcTlsError {
     #[error("Couldn't read file: {0}")]
     FileReadError(String),
     #[error("Error generating the certificate: {0}")]
-    CertGenerationError(#[from] RcgenError),
+    CertGenerationError(#[from] rcgen::Error),
     #[error("Error opening or writing the file: {0}")]
     IoError(#[from] io::Error),
 }
diff --git a/applications/minotari_app_utilities/Cargo.toml b/applications/minotari_app_utilities/Cargo.toml
index 3240758d25..8b05ee17ca 100644
--- a/applications/minotari_app_utilities/Cargo.toml
+++ b/applications/minotari_app_utilities/Cargo.toml
@@ -25,7 +25,6 @@ thiserror = "^1.0.26"
 dialoguer = { version = "0.10" }
 tonic = "0.12.3"
 
-
 [build-dependencies]
 tari_common = { path = "../../common", features = [
     "build",
diff --git a/scripts/cross_compile_ubuntu_18-pre-build.sh b/scripts/cross_compile_ubuntu_18-pre-build.sh
index e93f2a10fc..2292bd198f 100755
--- a/scripts/cross_compile_ubuntu_18-pre-build.sh
+++ b/scripts/cross_compile_ubuntu_18-pre-build.sh
@@ -5,40 +5,69 @@
 
 set -e
 
-USAGE="Usage: $0 target build ie: x86_64-unknown-linux-gnu or aarch64-unknown-linux-gnu"
+# APT Proxy for quicker testing
+#export HTTP_PROXY_APT=http://apt-proxy.local:3142
+if [ ! -z "${HTTP_PROXY_APT}" ] && [ -d "/etc/apt/apt.conf.d/" ]; then
+  echo "Setup apt proxy - ${HTTP_PROXY_APT}"
+  cat << APT-EoF > /etc/apt/apt.conf.d/proxy.conf
+Acquire {
+  HTTP::proxy "${HTTP_PROXY_APT}";
+  #HTTPS::proxy "http://127.0.0.1:8080";
+}
+APT-EoF
+fi
+
+USAGE="Usage: $0 {target build}
+ where target build is one of the following:
+  x86_64-unknown-linux-gnu or
+  aarch64-unknown-linux-gnu or
+  riscv64gc-unknown-linux-gnu
+"
 
 if [ "$#" == "0" ]; then
-  echo "$USAGE"
+  echo -e "${USAGE}"
   exit 1
 fi
 
 if [ -z "${CROSS_DEB_ARCH}" ]; then
-  echo "Should be run from cross, which sets the env CROSS_DEB_ARCH"
+  echo -e "Should be run from cross, which sets the env 'CROSS_DEB_ARCH'
+  amd64
+  arm64
+  riscv64
+"
   exit 1
 fi
 
+DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive}
+# Hack of Note!
+TimeZone=${TimeZone:-"Etc/GMT"}
+ln -snf /usr/share/zoneinfo/${TimeZone} /etc/localtime
+echo ${TimeZone} > /etc/timezone
+
 targetBuild="${1}"
 nativeRunTime=$(uname -m)
 echo "Native RunTime is ${nativeRunTime}"
 
 if [ "${nativeRunTime}" == "x86_64" ]; then
   nativeArch=amd64
-  if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then
-    targetArch=arm64
-    targetPlatform=aarch64
-  else
-    targetArch=amd64
-    targetPlatform=x86-64
-  fi
 elif [ "${nativeRunTime}" == "aarch64" ]; then
   nativeArch=arm64
-  if [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then
-    targetArch=amd64
-    targetPlatform=x86-64
-  fi
 elif [ "${nativeRunTime}" == "riscv64" ]; then
   nativeArch=riscv64
-  echo "ToDo!"
+else
+  echo "!!Unsupport platform!!"
+  exit 1
+fi
+
+if [ "${targetBuild}" == "aarch64-unknown-linux-gnu" ]; then
+  targetArch=arm64
+  targetPlatform=aarch64
+elif [ "${targetBuild}" == "x86_64-unknown-linux-gnu" ]; then
+  targetArch=amd64
+  targetPlatform=x86-64
+elif [ "${targetBuild}" == "riscv64gc-unknown-linux-gnu" ]; then
+  targetArch=riscv64
+  targetPlatform=riscv64
 else
   echo "!!Unsupport platform!!"
   exit 1
@@ -63,6 +92,7 @@ apt-get install --no-install-recommends --assume-yes \
   libsqlite3-0 \
   libreadline-dev \
   git \
+  make \
   cmake \
   dh-autoreconf \
   clang \
@@ -91,7 +121,7 @@ if [ "${CROSS_DEB_ARCH}" != "${nativeArch}" ]; then
   . /etc/lsb-release
   ubuntu_tag=${DISTRIB_CODENAME}
 
-  if [ "${crossArch}" == "arm64" ]; then
+  if [[ "${crossArch}" =~ ^(arm|riscv)64$ ]]; then
     cat << EoF > /etc/apt/sources.list.d/${ubuntu_tag}-${crossArch}.list
 deb [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse
 # deb-src [arch=${crossArch}] http://ports.ubuntu.com/ubuntu-ports ${ubuntu_tag} main restricted universe multiverse
@@ -143,24 +173,31 @@ deb [arch=amd64] http://security.ubuntu.com/ubuntu/ ${ubuntu_tag}-security multi
 EoF
   fi
 
+  dpkg --print-architecture
   dpkg --add-architecture ${CROSS_DEB_ARCH}
+  dpkg --print-architecture
   apt-get update
 
   # scripts/install_ubuntu_dependencies-cross_compile.sh x86-64
+#    pkg-config-${targetPlatform}-linux-gnu \
   apt-get --assume-yes install \
-    pkg-config-${targetPlatform}-linux-gnu \
     gcc-${targetPlatform}-linux-gnu \
     g++-${targetPlatform}-linux-gnu
 
   # packages needed for Ledger and hidapi
   apt-get --assume-yes install \
     libudev-dev:${CROSS_DEB_ARCH} \
-    libhidapi-dev:${CROSS_DEB_ARCH}
+    libhidapi-dev:${CROSS_DEB_ARCH} \
+    libssl-dev:${CROSS_DEB_ARCH}
 
 fi
 
+rustup show
+
 rustup target add ${targetBuild}
 rustup toolchain install stable-${targetBuild} --force-non-host
 
 rustup target list --installed
 rustup toolchain list
+
+rustup show
diff --git a/scripts/file_license_check.sh b/scripts/file_license_check.sh
index b33d814570..ce1feb1c75 100755
--- a/scripts/file_license_check.sh
+++ b/scripts/file_license_check.sh
@@ -1,5 +1,49 @@
-#!/bin/bash
+#!/usr/bin/env bash
+#
 # Must be run from the repo root
+#
+
+#set -xo pipefail
+set -e
+
+check_for() {
+  if prog_location=$(which ${1}) ; then
+    if result=$(${1} --version 2>/dev/null); then
+      result="${1}: ${result} INSTALLED ✓"
+    else
+      result="${1}: INSTALLED ✓"
+    fi
+  else
+    result="${1}: MISSING ⨯"
+  fi
+}
+
+check_requirements() {
+  echo "List of requirements and possible test:"
+  req_progs=(
+    mktemp
+    rg
+    diff
+  )
+  for RProg in "${req_progs[@]}"; do
+    check_for ${RProg}
+    echo "${result}"
+    if [[ "${result}" == "${RProg}: MISSING ⨯" ]]; then
+      echo "!! Install ${RProg} and try again !!"
+      exit -1
+    fi
+  done
+
+  if [ ! -f .license.ignore ]; then
+    echo "!! No .license.ignore file !!"
+    exit -1
+  fi
+}
+
+check_requirements
+
+diffparms=${diffparms:-"-u --suppress-blank-empty --strip-trailing-cr --color=never"}
+rgTemp=${rgTemp:-$(mktemp)}
 
 # rg -i "Copyright.*The Tari Project" --files-without-match \
 #    -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \
@@ -8,23 +52,31 @@
 # Exclude files without extensions as well as those with extensions that are not in the list
 #
 rg -i "Copyright.*The Tari Project" --files-without-match \
-    -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue}' . \
+    -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue,liquid,otf,d.ts,mjs}' . \
     | while IFS= read -r file; do
         if [[ -n $(basename "$file" | grep -E '\.') ]]; then
             echo "$file"
         fi
-    done | sort > /tmp/rgtemp
+    done | sort > ${rgTemp}
 
 # Sort the .license.ignore file as sorting seems to behave differently on different platforms
-cat .license.ignore | sort > /tmp/.license.ignore
+licenseIgnoreTemp=${licenseIgnoreTemp:-$(mktemp)}
+cat .license.ignore | sort > ${licenseIgnoreTemp}
+
+DIFFS=$( diff ${diffparms} ${licenseIgnoreTemp} ${rgTemp} || true )
 
-DIFFS=$(diff -u --strip-trailing-cr /tmp/.license.ignore /tmp/rgtemp)
+# clean up
+rm -vf ${rgTemp}
+rm -vf ${licenseIgnoreTemp}
 
-if [ -n "$DIFFS" ]; then
-    echo "New files detected that either need copyright/license identifiers added, or they need to be added to .license.ignore"
+if [ -n "${DIFFS}" ]; then
+    echo "New files detected that either need copyright/license identifiers added, "
+    echo "or they need to be added to .license.ignore"
     echo "NB: The ignore file must be sorted alphabetically!"
 
     echo "Diff:"
-    echo "$DIFFS"
+    echo "${DIFFS}"
     exit 1
+else
+    exit 0
 fi
diff --git a/scripts/install_ubuntu_dependencies.sh b/scripts/install_ubuntu_dependencies.sh
index b326d2207e..375b4e6166 100755
--- a/scripts/install_ubuntu_dependencies.sh
+++ b/scripts/install_ubuntu_dependencies.sh
@@ -12,6 +12,7 @@ apt-get install --no-install-recommends --assume-yes \
   libsqlite3-0 \
   libreadline-dev \
   git \
+  make \
   cmake \
   dh-autoreconf \
   clang \