From 599ab189771a45f376075897280b3fa77c550499 Mon Sep 17 00:00:00 2001 From: rzadp Date: Fri, 23 Aug 2024 10:31:32 +0200 Subject: [PATCH 1/4] Update and test the getting started script --- .github/workflows/check-getting-started.yml | 296 ++++++++++++++++++++ scripts/getting-started.sh | 110 +++++--- 2 files changed, 367 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/check-getting-started.yml diff --git a/.github/workflows/check-getting-started.yml b/.github/workflows/check-getting-started.yml new file mode 100644 index 000000000000..82f31cdefcbb --- /dev/null +++ b/.github/workflows/check-getting-started.yml @@ -0,0 +1,296 @@ +name: Check the getting-started.sh script + +# This workflow aims to make sure that the `getting-started.sh` script +# is functional and allows to build the templates +# on different operating systems. +# +# There are two jobs inside. +# One for systems that can run in a docker container, and one for macOS. +# +# Each job consists of: +# 1. Some necessary prerequisites for the workflow itself. +# 2. A first pass of the script, which will install dependencies and clone a template. +# 3. A second pass of the script, to make sure the behaviour is as expected. +# 4. Building the template - making sure it's buildable and runnable. +# +# The script is interacted with using the `expect` tool, which is available on all relevant systems. +# The steps are not re-used between macOS and other systems, +# because they are very similar but a little different. +# Additionally, macOS does NOT start from scratch here - for example, we have homebrew already installed. +# +# There are many combinations of systems, shells and templates. +# We test a selected handful of combinations here. + +on: + push: + paths: + - '.github/workflows/check-getting-started.yml' + - 'scripts/getting-started.sh' + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check-getting-started: + strategy: + fail-fast: true + matrix: + include: + - name: ubuntu + container: ubuntu + template: minimal + shell: bash + - name: debian + container: debian + template: parachain + shell: sh + - name: arch + container: archlinux + template: solochain + shell: sh + - name: fedora + container: fedora + template: parachain + shell: sh + - name: opensuse + container: opensuse/tumbleweed + template: solochain + shell: sh + runs-on: arc-runners-polkadot-sdk-beefy + container: ${{ matrix.container }}:latest + steps: + # A minimal amount of prerequisites required before we can run the actual getting-started script, + # which will install the rest of requirements. + - name: Install ubuntu/debian prerequisites + run: apt update && apt install -y expect sudo git + if: contains(matrix.name, 'ubuntu') || contains(matrix.name, 'debian') + - name: Install arch prerequisites + run: pacman -Syu --needed --noconfirm expect sudo git + if: contains(matrix.name, 'arch') + - name: Install fedora prerequisites + run: dnf --assumeyes install expect sudo git + if: contains(matrix.name, 'fedora') + - name: Install opensuse prerequisites + run: zypper install --no-confirm expect sudo git + if: contains(matrix.name, 'opensuse') + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set additional expect flags if necessary + run: | + # Add a debug flag to expect, if github is re-run with debug logging enabled. + [ "${{ runner.debug }}" = "1" ] && EXPECT_FLAGS="-d" || EXPECT_FLAGS="" + echo "EXPECT_FLAGS=${EXPECT_FLAGS}" >> $GITHUB_ENV + + - name: Check the first run if the script + run: | + expect $EXPECT_FLAGS -c ' + set timeout 240 + + spawn ${{ matrix.shell }} scripts/getting-started.sh + + expect_after { + timeout { puts stderr "Timed out on an expect"; exit 1 } + eof { puts stderr "EOF received on an expect"; exit 1 } + } + + expect -nocase "Detected ${{ matrix.name }}" + + expect "Rust is not installed. Install it?" { + send "y\r" + expect "Proceed with standard installation (default - just press enter)" { + send "\r" + expect "Rust is installed now" + } + } + + expect "Setup the Rust environment" { + send "y\r" + } + + expect "start with one of the templates" { + send "y\r" + } + + expect -re "(.)\\) ${{ matrix.template }} template" { + send "$expect_out(1,string)\r" + } + + expect "compile the node?" { + send "n\r" + } + + expect eof + ' + timeout-minutes: 15 + + - name: Check the second run if the script + run: | + expect $EXPECT_FLAGS -c ' + set timeout 120 + + spawn ${{ matrix.shell }} scripts/getting-started.sh + + expect_after { + timeout { puts stderr "Timed out on an expect"; exit 1 } + eof { puts stderr "EOF received on an expect"; exit 1 } + } + + expect "Rust already installed" {} + + expect "Setup the Rust environment" { + send "n\r" + } + + expect "start with one of the templates" { + send "y\r" + } + + expect -re "(.)\\) ${{ matrix.template }} template" { + send "$expect_out(1,string)\r" + expect "directory already exists" {} + } + + expect "compile the node?" { + send "n\r" + } + + expect eof + ' + timeout-minutes: 15 + + - name: Compile the node outside of the script + run: | + . "$HOME/.cargo/env" + cd ${{ matrix.template }}-template + cargo build --release + timeout-minutes: 120 + + - name: Check that the binary is executable + run: | + . "$HOME/.cargo/env" + cd ${{ matrix.template }}-template + cargo run --release -- --help + timeout-minutes: 5 + + check-getting-started-macos: + strategy: + fail-fast: true + matrix: + include: + - template: parachain + shell: sh + - template: solochain + shell: bash + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set additional expect flags if necessary + run: | + # Add a debug flag to expect, if github is re-run with debug logging enabled. + [ "${{ runner.debug }}" = "1" ] && EXPECT_FLAGS="-d" || EXPECT_FLAGS="" + echo "EXPECT_FLAGS=${EXPECT_FLAGS}" >> $GITHUB_ENV + + - name: Check the first run if the script + run: | + expect $EXPECT_FLAGS -c ' + set timeout 120 + + spawn ${{ matrix.shell }} scripts/getting-started.sh + + expect_after { + timeout { puts stderr "Timed out on an expect"; exit 1 } + eof { puts stderr "EOF received on an expect"; exit 1 } + } + + expect -nocase "Detected macOS" + + expect "Homebrew already installed" + + expect "Install cmake" { + send "y\r" + } + + expect "Rust already installed" {} + + expect "Setup the Rust environment" { + send "y\r" + } + + expect "start with one of the templates" { + send "y\r" + } + + expect -re "(.)\\) ${{ matrix.template }} template" { + send "$expect_out(1,string)\r" + } + + expect "compile the node?" { + send "n\r" + } + + expect eof + ' + timeout-minutes: 15 + + - name: Check the second run if the script + run: | + expect $EXPECT_FLAGS -c ' + set timeout 120 + + spawn ${{ matrix.shell }} scripts/getting-started.sh + + expect_after { + timeout { puts stderr "Timed out on an expect"; exit 1 } + eof { puts stderr "EOF received on an expect"; exit 1 } + } + + expect "Homebrew already installed" + + expect "Install cmake" { + send "y\r" + } + + expect "Rust already installed" {} + + expect "Setup the Rust environment" { + send "n\r" + } + + expect "start with one of the templates" { + send "y\r" + } + + expect -re "(.)\\) ${{ matrix.template }} template" { + send "$expect_out(1,string)\r" + expect "directory already exists" {} + } + + expect "compile the node?" { + send "n\r" + } + + expect eof + ' + timeout-minutes: 15 + + - name: Compile the node outside of the script + run: | + . "$HOME/.cargo/env" + cd ${{ matrix.template }}-template + cargo build --release + timeout-minutes: 120 + + - name: Check that the binary is executable + run: | + . "$HOME/.cargo/env" + cd ${{ matrix.template }}-template + cargo run --release -- --help + timeout-minutes: 5 diff --git a/scripts/getting-started.sh b/scripts/getting-started.sh index 57806280914b..d5fd545481d1 100755 --- a/scripts/getting-started.sh +++ b/scripts/getting-started.sh @@ -4,30 +4,41 @@ set -e prompt() { while true; do - echo "$1 [y/N]" + printf "$1 [y/N]\n" read yn case $yn in [Yy]* ) return 0;; # Yes, return 0 (true) [Nn]* ) return 1;; # No, return 1 (false) "" ) return 1;; # Default to no if user just presses Enter - * ) echo "Please answer yes or no.";; + * ) printf "Please answer yes or no.\n";; esac done } prompt_default_yes() { while true; do - echo "$1 [Y/n]" + printf "$1 [Y/n]\n" read yn case $yn in [Yy]* ) return 0;; # Yes, return 0 (true) [Nn]* ) return 1;; # No, return 1 (false) "" ) return 0;; # Default to yes if user just presses Enter - * ) echo "Please answer yes or no.";; + * ) printf "Please answer yes or no.\n";; esac done } +clone_and_enter_template() { + template="$1" # minimal, solochain, or parachain + if [ -d "${template}-template" ]; then + printf "\n✅︎ ${template}-template directory already exists. -> Entering.\n" + else + printf "\n↓ Let's grab the ${template} template from github.\n" + git clone --quiet https://github.com/paritytech/polkadot-sdk-${template}-template.git ${template}-template + fi + cd ${template}-template +} + cat </dev/null 2>&1; then - echo "\n✅︎🍺 Homebrew already installed." + printf "\n✅︎🍺 Homebrew already installed.\n" else - if prompt_default_yes "\n🍺 Homebrew is not installed. Install it?"; then - echo "🍺 Installing Homebrew." + if prompt_default_yes "\n🍺 Homebrew is not installed. Install it?\n"; then + printf "🍺 Installing Homebrew.\n" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" else - echo "❌ Cannot continue without homebrew. Aborting." + printf "❌ Cannot continue without homebrew. Aborting.\n" exit 1 fi fi brew update if command -v git >/dev/null 2>&1; then - echo "\n✅︎🍺 git already installed." + printf "\n✅︎🍺 git already installed.\n" else - if prompt_default_yes "\n🍺 git seems to be missing but we will need it; install git?"; then + if prompt_default_yes "\n🍺 git seems to be missing but we will need it; install git?\n"; then brew install git else - echo "❌ Cannot continue without git. Aborting." + printf "❌ Cannot continue without git. Aborting.\n" exit 1 fi fi @@ -75,73 +86,94 @@ if [ "$os_name" = "Darwin" ]; then if prompt "\n🍺 Install cmake, openssl and protobuf?"; then brew install cmake openssl protobuf else - echo "🍺 Assuming cmake, openssl and protobuf are present." + printf "🍺 Assuming cmake, openssl and protobuf are present.\n" fi elif [ "$os_name" = "Linux" ]; then # find the distro name in the release files distro=$( cat /etc/*-release | tr '[:upper:]' '[:lower:]' | grep -Poi '(debian|ubuntu|arch|fedora|opensuse)' | uniq | head -n 1 ) if [ "$distro" = "ubuntu" ]; then - echo "\n🐧 Detected Ubuntu. Using apt to install dependencies." - sudo apt install --assume-yes git clang curl libssl-dev protobuf-compiler + printf "\n🐧 Detected Ubuntu. Using apt to install dependencies.\n" + sudo apt -qq update + sudo apt -qq install --assume-yes git clang curl libssl-dev protobuf-compiler make elif [ "$distro" = "debian" ]; then - echo "\n🐧 Detected Debian. Using apt to install dependencies." - sudo apt install --assume-yes git clang curl libssl-dev llvm libudev-dev make protobuf-compiler + printf "\n🐧 Detected Debian. Using apt to install dependencies.\n" + sudo apt -qq update + sudo apt -qq install --assume-yes git clang curl libssl-dev llvm libudev-dev make protobuf-compiler elif [ "$distro" = "arch" ]; then - echo "\n🐧 Detected Arch Linux. Using pacman to install dependencies." + printf "\n🐧 Detected Arch Linux. Using pacman to install dependencies.\n" pacman -Syu --needed --noconfirm curl git clang make protobuf elif [ "$distro" = "fedora" ]; then - echo "\n🐧 Detected Fedora. Using dnf to install dependencies." - sudo dnf update - sudo dnf install clang curl git openssl-devel make protobuf-compiler + printf "\n🐧 Detected Fedora. Using dnf to install dependencies.\n" + sudo dnf update --assumeyes + sudo dnf install --assumeyes clang curl git openssl-devel make protobuf-compiler perl elif [ "$distro" = "opensuse" ]; then - echo "\n🐧 Detected openSUSE. Using zypper to install dependencies." - sudo zypper install clang curl git openssl-devel llvm-devel libudev-devel make protobuf + printf "\n🐧 Detected openSUSE. Using zypper to install dependencies.\n" + sudo zypper install --no-confirm clang gcc gcc-c++ curl git openssl-devel llvm-devel libudev-devel make awk protobuf-devel else - if prompt "\n🐧 Unknown Linux distribution. Unable to install dependencies. Continue anyway?"; then - echo "\n🐧 Proceeding with unknown linux distribution..." + if prompt "\n🐧 Unknown Linux distribution. Unable to install dependencies. Continue anyway?\n"; then + printf "\n🐧 Proceeding with unknown linux distribution...\n" else exit 1 fi fi else - echo "❌ Unknown operating system. Aborting." + printf "❌ Unknown operating system. Aborting.\n" exit 1 fi # Check if rust is installed +[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env" if command -v rustc >/dev/null 2>&1; then - echo "\n✅︎🦀 Rust already installed." + printf "\n✅︎🦀 Rust already installed.\n" else if prompt_default_yes "\n🦀 Rust is not installed. Install it?"; then - echo "🦀 Installing via rustup." + printf "🦀 Installing via rustup.\n" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + . "$HOME/.cargo/env" else - echo "Aborting." + printf "Aborting.\n" exit 1 fi fi # Ensure that we have wasm support if prompt_default_yes "\n🦀 Setup the Rust environment (e.g. WASM support)?"; then - echo "🦀 Setting up Rust environment." + printf "🦀 Setting up Rust environment.\n" rustup default stable rustup update rustup target add wasm32-unknown-unknown rustup component add rust-src fi -if [ -d "minimal-template" ]; then - echo "\n✅︎ minimal-template directory already exists. -> Entering." -else - echo "\n↓ Let's grab the minimal template from github." - git clone https://github.com/paritytech/polkadot-sdk-minimal-template.git minimal-template +if ! prompt "\nWould you like to start with one of the templates?"; then + printf "⚡ All done, the environment is ready for hacking.\n" + exit 0 +fi + +while true; do + printf "\nWhich template would you like to start with?\n" + printf "1) minimal template\n" + printf "2) parachain template\n" + printf "3) solochain template\n" + printf "q) cancel\n" + read -p "#? " template + case $template in + [1]* ) clone_and_enter_template minimal; break;; + [2]* ) clone_and_enter_template parachain; break;; + [3]* ) clone_and_enter_template solochain; break;; + [qQ]* ) printf "Canceling, not using a template.\n"; exit 0;; + * ) printf "Selection not recognized.\n";; + esac +done + +if ! prompt_default_yes "\n⚙️ Let's compile the node? It might take a while."; then + printf "⚡ Script finished, you can continue in the ${template}-template directory.\n" + exit 0 fi -cd minimal-template -echo "\n⚙️ Let's compile the node." cargo build --release -if prompt_default_yes "\n🚀 Everything ready to go, let's run the node?"; then +if prompt_default_yes "\n🚀 Everything ready to go, let's run the node?\n"; then cargo run --release -- --dev fi From bc9e25ce8024d7f982bef7c8140c64c47c426f72 Mon Sep 17 00:00:00 2001 From: rzadp Date: Fri, 23 Aug 2024 11:16:32 +0200 Subject: [PATCH 2/4] run on PR --- .github/workflows/check-getting-started.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-getting-started.yml b/.github/workflows/check-getting-started.yml index 82f31cdefcbb..c9bc5c83b84d 100644 --- a/.github/workflows/check-getting-started.yml +++ b/.github/workflows/check-getting-started.yml @@ -22,7 +22,7 @@ name: Check the getting-started.sh script # We test a selected handful of combinations here. on: - push: + pull_request: paths: - '.github/workflows/check-getting-started.yml' - 'scripts/getting-started.sh' From 7ebc55024121e22e4fbd9f3d7d4fecd2c29e25a5 Mon Sep 17 00:00:00 2001 From: Przemek Rzad Date: Sat, 31 Aug 2024 14:15:57 +0200 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Oliver Tale-Yazdi --- .github/workflows/check-getting-started.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-getting-started.yml b/.github/workflows/check-getting-started.yml index c9bc5c83b84d..acb216cbf9d4 100644 --- a/.github/workflows/check-getting-started.yml +++ b/.github/workflows/check-getting-started.yml @@ -87,7 +87,7 @@ jobs: [ "${{ runner.debug }}" = "1" ] && EXPECT_FLAGS="-d" || EXPECT_FLAGS="" echo "EXPECT_FLAGS=${EXPECT_FLAGS}" >> $GITHUB_ENV - - name: Check the first run if the script + - name: Check the first run of the script run: | expect $EXPECT_FLAGS -c ' set timeout 240 @@ -129,7 +129,7 @@ jobs: ' timeout-minutes: 15 - - name: Check the second run if the script + - name: Check the second run of the script run: | expect $EXPECT_FLAGS -c ' set timeout 120 @@ -240,7 +240,7 @@ jobs: ' timeout-minutes: 15 - - name: Check the second run if the script + - name: Check the second run of the script run: | expect $EXPECT_FLAGS -c ' set timeout 120 From 3dabc12ad7d999468372591c83c91062b3e14b79 Mon Sep 17 00:00:00 2001 From: Przemek Rzad Date: Sat, 31 Aug 2024 15:13:37 +0200 Subject: [PATCH 4/4] Update .github/workflows/check-getting-started.yml --- .github/workflows/check-getting-started.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-getting-started.yml b/.github/workflows/check-getting-started.yml index acb216cbf9d4..b43db33c63bf 100644 --- a/.github/workflows/check-getting-started.yml +++ b/.github/workflows/check-getting-started.yml @@ -198,7 +198,7 @@ jobs: [ "${{ runner.debug }}" = "1" ] && EXPECT_FLAGS="-d" || EXPECT_FLAGS="" echo "EXPECT_FLAGS=${EXPECT_FLAGS}" >> $GITHUB_ENV - - name: Check the first run if the script + - name: Check the first run of the script run: | expect $EXPECT_FLAGS -c ' set timeout 120