From d5675580fc6d7c5722b6762604da30afa1ed9318 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 12:36:39 +0100 Subject: [PATCH 01/10] Publish release with wasms --- .circleci/config.yml | 3 +++ cpp/scripts/publish_release | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100755 cpp/scripts/publish_release diff --git a/.circleci/config.yml b/.circleci/config.yml index 28e83ba7..5b021af2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,6 +100,9 @@ jobs: - run: name: 'Build' command: cond_spot_run_build aztec3-circuits-wasm-linux-clang 64 + - run: + name: 'Publish' + command: scripts/publish_release aztec3-circuits-wasm-linux-clang wasm-linux-clang-assert: docker: diff --git a/cpp/scripts/publish_release b/cpp/scripts/publish_release new file mode 100755 index 00000000..636009c4 --- /dev/null +++ b/cpp/scripts/publish_release @@ -0,0 +1,35 @@ +#!/bin/bash +set -euo pipefail + +# Publishes a release to github (depends on build-system) + +REPOSITORY=$1 +PROJECT_DIR=$(query_manifest projectDir $REPOSITORY) +EXTRACT_DIR=../extracted/$REPOSITORY +ARTIFACTS_DIR=../artifacts + +echo "publish_release: Repository: $REPOSITORY" +echo "publish_release: Project Dir: $PROJECT_DIR" +echo "publish_release: Working directory: $PWD" +echo "publish_release: Extract dir: $EXTRACT_DIR" +echo "publish_release: Artifacts dir: $ARTIFACTS" + +# Get the contents of the build +extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR +echo "publish_release: Extracted $(ls $EXTRACT_DIR)" + +# Prepare the artifacts to push +mkdir -p $ARTIFACTS_DIR +# TODO: Make these params of the script and not hardcode them here +cp $EXTRACT_DIR/build-wasm/bin/aztec3-circuits.wasm $EXTRACT_DIR/barretenberg/cpp/build-wasm/bin/primitives.wasm $ARTIFACTS_DIR/ + +# Version to use for the release +# TODO: if this is a tagged release, just use the tag and drop the commit hash +VERSION=$(cat VERSION)-$COMMIT_HASH + +echo "publish_release: Publishing $VERSION" +echo "publish_release: $(ls $ARTIFACTS_DIR/)" + +# Get ghr to make the release and create it +go get github.com/tcnksm/ghr +ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${COMMIT_HASH} -delete ${VERSION} $ARTIFACTS_DIR/ From 47df28b9c0770095431819987d02d0058ef78975 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 12:38:50 +0100 Subject: [PATCH 02/10] Move publish script --- .circleci/config.yml | 2 +- {cpp/scripts => scripts}/publish_release | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {cpp/scripts => scripts}/publish_release (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b021af2..3aadb962 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,7 +102,7 @@ jobs: command: cond_spot_run_build aztec3-circuits-wasm-linux-clang 64 - run: name: 'Publish' - command: scripts/publish_release aztec3-circuits-wasm-linux-clang + command: ./scripts/publish_release aztec3-circuits-wasm-linux-clang wasm-linux-clang-assert: docker: diff --git a/cpp/scripts/publish_release b/scripts/publish_release similarity index 100% rename from cpp/scripts/publish_release rename to scripts/publish_release From e33cf81461639d089de0548adbec8b877fb6751b Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 12:40:10 +0100 Subject: [PATCH 03/10] Fix script 1 --- scripts/publish_release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/publish_release b/scripts/publish_release index 636009c4..782ab4bf 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -12,7 +12,7 @@ echo "publish_release: Repository: $REPOSITORY" echo "publish_release: Project Dir: $PROJECT_DIR" echo "publish_release: Working directory: $PWD" echo "publish_release: Extract dir: $EXTRACT_DIR" -echo "publish_release: Artifacts dir: $ARTIFACTS" +echo "publish_release: Artifacts dir: $ARTIFACTS_DIR" # Get the contents of the build extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR From 811c5d8ea96b658d5d6f3ba878a0d6d745a18b96 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 12:45:28 +0100 Subject: [PATCH 04/10] Fix script 2 --- scripts/publish_release | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/publish_release b/scripts/publish_release index 782ab4bf..3b08115a 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -15,8 +15,22 @@ echo "publish_release: Extract dir: $EXTRACT_DIR" echo "publish_release: Artifacts dir: $ARTIFACTS_DIR" # Get the contents of the build -extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR -echo "publish_release: Extracted $(ls $EXTRACT_DIR)" +# extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR +IMAGE_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH +EXTRACT_FROM=/usr/src/$PROJECT_DIR +EXTRACT_TO=$EXTRACT_DIR + +echo "Pulling $IMAGE_COMMIT_URI..." +docker pull $IMAGE_COMMIT_URI +TEMP_CONTAINER=$(docker create $IMAGE_COMMIT_URI) + +echo "Extracting $EXTRACT_FROM from $REPOSITORY to $EXTRACT_TO..." +mkdir -p $EXTRACT_TO +docker cp $TEMP_CONTAINER:$EXTRACT_FROM $EXTRACT_TO +docker rm -v $TEMP_CONTAINER > /dev/null + +echo "Extracted contents:" +ls -al $EXTRACT_TO # Prepare the artifacts to push mkdir -p $ARTIFACTS_DIR From afb37bffc7ee5716fa32324703437fe8fab795b4 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:23:13 +0100 Subject: [PATCH 05/10] Install docker --- scripts/publish_release | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/publish_release b/scripts/publish_release index 3b08115a..52d5e903 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -14,6 +14,10 @@ echo "publish_release: Working directory: $PWD" echo "publish_release: Extract dir: $EXTRACT_DIR" echo "publish_release: Artifacts dir: $ARTIFACTS_DIR" +# Ensure we got docker in this image +apk add --update docker openrc +echo "publish_release: Docker version: $(docker --version)" + # Get the contents of the build # extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR IMAGE_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH From d57cbddacc5059f8d6f697cd5ea0baa66bcacbf0 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:24:57 +0100 Subject: [PATCH 06/10] Install golang --- scripts/publish_release | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/publish_release b/scripts/publish_release index 52d5e903..909c284b 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -45,9 +45,12 @@ cp $EXTRACT_DIR/build-wasm/bin/aztec3-circuits.wasm $EXTRACT_DIR/barretenberg/cp # TODO: if this is a tagged release, just use the tag and drop the commit hash VERSION=$(cat VERSION)-$COMMIT_HASH -echo "publish_release: Publishing $VERSION" -echo "publish_release: $(ls $ARTIFACTS_DIR/)" +# Get ghr to make the release and create it, installing golang if needed +apk add --no-cache git make musl-dev go +echo "publish_release: Golang version: $(go --version)" -# Get ghr to make the release and create it go get github.com/tcnksm/ghr + +echo "publish_release: Publishing $VERSION" +echo "publish_release: $(ls $ARTIFACTS_DIR/)" ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${COMMIT_HASH} -delete ${VERSION} $ARTIFACTS_DIR/ From d2df9739d34a434dd4f0cc59baafa4fc19d66f90 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:28:05 +0100 Subject: [PATCH 07/10] Use circle base img --- .circleci/config.yml | 4 ++-- scripts/publish_release | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3aadb962..37d664a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,7 +92,7 @@ save_logs: &save_logs jobs: wasm-linux-clang: docker: - - image: aztecprotocol/alpine-build-image + - image: cimg/base:2023.04 resource_class: small steps: - *checkout @@ -119,7 +119,7 @@ jobs: docker: - image: aztecprotocol/alpine-build-image resource_class: small - steps: + steps: - *checkout - *setup_env - run: diff --git a/scripts/publish_release b/scripts/publish_release index 909c284b..16fce748 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -14,10 +14,6 @@ echo "publish_release: Working directory: $PWD" echo "publish_release: Extract dir: $EXTRACT_DIR" echo "publish_release: Artifacts dir: $ARTIFACTS_DIR" -# Ensure we got docker in this image -apk add --update docker openrc -echo "publish_release: Docker version: $(docker --version)" - # Get the contents of the build # extract_repo $REPOSITORY /usr/src/$PROJECT_DIR $EXTRACT_DIR IMAGE_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH @@ -45,10 +41,7 @@ cp $EXTRACT_DIR/build-wasm/bin/aztec3-circuits.wasm $EXTRACT_DIR/barretenberg/cp # TODO: if this is a tagged release, just use the tag and drop the commit hash VERSION=$(cat VERSION)-$COMMIT_HASH -# Get ghr to make the release and create it, installing golang if needed -apk add --no-cache git make musl-dev go -echo "publish_release: Golang version: $(go --version)" - +# Get ghr to make the release and create it go get github.com/tcnksm/ghr echo "publish_release: Publishing $VERSION" From 83ff4d50329114f85c194e51a11d2f6378812708 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:34:23 +0100 Subject: [PATCH 08/10] Install deps in circle base img --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 37d664a6..509bdc47 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,6 +95,9 @@ jobs: - image: cimg/base:2023.04 resource_class: small steps: + - run: + name: Install deps + command: pip3 install awscli - *checkout - *setup_env - run: From f037033f9c75e59a02d108c57eefc59c927d09de Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:48:40 +0100 Subject: [PATCH 09/10] Use aws cli base img --- .circleci/config.yml | 4 ++-- scripts/publish_release | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 509bdc47..d1db74df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,12 +92,12 @@ save_logs: &save_logs jobs: wasm-linux-clang: docker: - - image: cimg/base:2023.04 + - image: cimg/aws:2023.04 resource_class: small steps: - run: name: Install deps - command: pip3 install awscli + command: sudo apt update && sudo apt install golang-go && go install github.com/tcnksm/ghr@latest - *checkout - *setup_env - run: diff --git a/scripts/publish_release b/scripts/publish_release index 16fce748..0851eac3 100755 --- a/scripts/publish_release +++ b/scripts/publish_release @@ -42,8 +42,6 @@ cp $EXTRACT_DIR/build-wasm/bin/aztec3-circuits.wasm $EXTRACT_DIR/barretenberg/cp VERSION=$(cat VERSION)-$COMMIT_HASH # Get ghr to make the release and create it -go get github.com/tcnksm/ghr - echo "publish_release: Publishing $VERSION" echo "publish_release: $(ls $ARTIFACTS_DIR/)" -ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${COMMIT_HASH} -delete ${VERSION} $ARTIFACTS_DIR/ +~/go/bin/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${COMMIT_HASH} -delete ${VERSION} $ARTIFACTS_DIR/ From 77f1de05b7797ef4efc47d44b97b739f36cf67d1 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 6 Apr 2023 14:53:26 +0100 Subject: [PATCH 10/10] Start docker --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1db74df..1356ccde 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,6 +103,8 @@ jobs: - run: name: 'Build' command: cond_spot_run_build aztec3-circuits-wasm-linux-clang 64 + - setup_remote_docker: + docker_layer_caching: true - run: name: 'Publish' command: ./scripts/publish_release aztec3-circuits-wasm-linux-clang