Various CI fixes (#3813)

## Issue Addressed Closes #3812 Closes #3750 Closes #3705
sigp · Dec 20, 2022 · cb1eb76 · cb1eb76
1 parent ffbf70e
commit cb1eb76
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 37 deletions.
diff --git a/.github/workflows/local-testnet.yml b/.github/workflows/local-testnet.yml
@@ -21,7 +21,7 @@ jobs:
       - name: Get latest version of stable Rust
         run: rustup update stable
       - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
         with:
             repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install ganache

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,8 +8,8 @@ on:
 env:
     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-    REPO_NAME: sigp/lighthouse
-    IMAGE_NAME: sigp/lighthouse
+    REPO_NAME: ${{ github.repository_owner }}/lighthouse
+    IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
 
 jobs:
     extract-version:
@@ -63,12 +63,8 @@ jobs:
         steps:
             - name: Checkout sources
               uses: actions/checkout@v3
-            - name: Build toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  profile:  minimal
-                  override: true
+            - name: Get latest version of stable Rust
+              run: rustup update stable
 
             # ==============================
             # Windows dependencies
@@ -88,7 +84,7 @@ jobs:
             # ==============================
             - name: Install Protoc
               if: contains(matrix.arch, 'darwin') || contains(matrix.arch, 'windows')
-              uses: arduino/setup-protoc@v1
+              uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
               with:
                 repo-token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -179,13 +175,13 @@ jobs:
             # =======================================================================
 
             - name:  Upload artifact
-              uses:  actions/upload-artifact@v2
+              uses:  actions/upload-artifact@v3
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
 
             - name: Upload signature
-              uses: actions/upload-artifact@v2
+              uses: actions/upload-artifact@v3
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
@@ -208,19 +204,22 @@ jobs:
             # ==============================
 
             - name: Download artifacts
-              uses: actions/download-artifact@v2
+              uses: actions/download-artifact@v3
 
             # ==============================
             #       Create release draft
             # ==============================
 
             - name: Generate Full Changelog
               id: changelog
-              run: echo "CHANGELOG=$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+              run: |
+                echo "CHANGELOG<<EOF" >> $GITHUB_OUTPUT
+                echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+                echo "EOF" >> $GITHUB_OUTPUT
 
             - name: Create Release Draft
               env:
-                  GITHUB_USER: sigp
+                  GITHUB_USER: ${{ github.repository_owner }}
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
               # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml

diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
@@ -13,6 +13,8 @@ env:
   RUSTFLAGS: "-D warnings"
   # The Nightly version used for cargo-udeps, might need updating from time to time.
   PINNED_NIGHTLY: nightly-2022-05-20
+  # Prevent Github API rate limiting.
+  LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   target-branch-check:
     name: target-branch-check
@@ -51,7 +53,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -95,7 +97,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run beacon_chain tests for all known forks
@@ -109,7 +111,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run operation_pool tests for all known forks
@@ -133,7 +135,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -149,7 +151,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run state_transition_vectors in release.
@@ -163,7 +165,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run consensus-spec-tests with blst, milagro and fake_crypto
@@ -189,7 +191,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -205,7 +207,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -221,7 +223,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -237,7 +239,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -253,7 +255,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install ganache
@@ -285,7 +287,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run exec engine integration tests in release
@@ -299,7 +301,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Typecheck benchmark code without running it
@@ -323,7 +325,7 @@ jobs:
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Lint code for quality and style with Clippy
@@ -346,7 +348,7 @@ jobs:
         cargo build --release --bin cargo-clippy --bin clippy-driver
         cargo build --release --bin cargo-clippy --bin clippy-driver -Zunstable-options --out-dir $(rustc --print=sysroot)/bin
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run Clippy with the disallowed-from-async lint
@@ -360,7 +362,7 @@ jobs:
     - name: Install Rust @ MSRV (${{ needs.extract-msrv.outputs.MSRV }})
       run: rustup override set ${{ needs.extract-msrv.outputs.MSRV }}
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run cargo check
@@ -404,7 +406,7 @@ jobs:
     # NOTE: cargo-udeps version is pinned until this issue is resolved:
     # https://github.com/est31/cargo-udeps/issues/135
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install cargo-udeps

diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile
@@ -13,6 +13,8 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)
 
+WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget)
+
 all:
 	make $(OUTPUT_DIR)
 	make $(BLS_OUTPUT_DIR)
@@ -25,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS)
 
 $(BLS_OUTPUT_DIR):
 	mkdir $(BLS_OUTPUT_DIR)
-	wget $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
+	$(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
 	tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR)
 
 %-$(TESTS_TAG).tar.gz:
-	wget $(BASE_URL)/$*.tar.gz -O $@
+	$(WGET) $(BASE_URL)/$*.tar.gz -O $@
 
 clean-test-files:
 	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)

diff --git a/testing/web3signer_tests/build.rs b/testing/web3signer_tests/build.rs
@@ -1,7 +1,10 @@
 //! This build script downloads the latest Web3Signer release and places it in the `OUT_DIR` so it
 //! can be used for integration testing.
 
-use reqwest::Client;
+use reqwest::{
+    header::{self, HeaderValue},
+    Client,
+};
 use serde_json::Value;
 use std::env;
 use std::fs;
@@ -15,10 +18,15 @@ const FIXED_VERSION_STRING: Option<&str> = None;
 #[tokio::main]
 async fn main() {
     let out_dir = env::var("OUT_DIR").unwrap();
-    download_binary(out_dir.into()).await;
+
+    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
+    // We use a name that is unlikely to accidentally collide with anything the user has configured.
+    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
+
+    download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await;
 }
 
-pub async fn download_binary(dest_dir: PathBuf) {
+pub async fn download_binary(dest_dir: PathBuf, github_token: &str) {
     let version_file = dest_dir.join("version");
 
     let client = Client::builder()
@@ -33,8 +41,11 @@ pub async fn download_binary(dest_dir: PathBuf) {
         env_version
     } else {
         // Get the latest release of the web3 signer repo.
+        let mut token_header_value = HeaderValue::from_str(github_token).unwrap();
+        token_header_value.set_sensitive(true);
         let latest_response: Value = client
             .get("https://api.github.com/repos/ConsenSys/web3signer/releases/latest")
+            .header(header::AUTHORIZATION, token_header_value)
             .send()
             .await
             .unwrap()