From 8b8c9968a5c307c46d5f5940c62dad66c738e134 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 31 Mar 2023 12:45:45 -0700 Subject: [PATCH 01/25] Try caching images instead of build layers --- multidim-interop/dockerBuildWrapper.sh | 40 ++++++++++++++++++++++---- multidim-interop/go/v0.22/Dockerfile | 5 +++- multidim-interop/go/v0.23/Dockerfile | 5 +++- multidim-interop/go/v0.24/Dockerfile | 5 +++- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/multidim-interop/dockerBuildWrapper.sh b/multidim-interop/dockerBuildWrapper.sh index 087a4e8c9..7d62c1a3f 100755 --- a/multidim-interop/dockerBuildWrapper.sh +++ b/multidim-interop/dockerBuildWrapper.sh @@ -1,13 +1,43 @@ #!/usr/bin/env /bin/bash +set -eou pipefail +set -x +# Print the hash of our current working directory. +# We'll use this to store the build cache in S3. +CWD_HASH=$(tar --sort=name \ + --mtime="1970-01-01 00:00Z" \ + --owner=0 --group=0 \ + -cvf - . | sha256sum | sed 's/.$//' | xargs) # Sed removes the trailing dash, xargs removes the spaces +ARCH=$(docker info -f "{{.Architecture}}") +BUILD_HASH="$IMAGE_NAME-$CWD_HASH-$ARCH" + +echo "Build hash is: $BUILD_HASH" + +# If we don't have this image, and we have an S3 bucket defined, try to load it from S3. +# If we have this image, it's probably faster to rebuild the parts that have changed rather than fetch from S3. +if (! docker image inspect $IMAGE_NAME -f "{{.Id}}") && [[ -n "${AWS_BUCKET}" ]]; then + echo "Trying to load image from S3" + aws s3 cp s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz - | docker image load && \ + echo "Loaded image from S3" && exit 0 \ + || echo "Failed to load image from S3" +fi + +# TODO maybe get rid of this caching mode CACHING_OPTIONS="" # If in CI and we have a defined cache bucket, use caching -if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then - CACHING_OPTIONS="\ - --cache-to type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME \ - --cache-from type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME" -fi +# if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then + # CACHING_OPTIONS="\ + # --cache-to type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME \ + # --cache-from type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME" +# fi +# echo "Caching options: $CI" docker buildx build \ --load \ -t $IMAGE_NAME $CACHING_OPTIONS "$@" + +# If we're in CI, let's save the image to S3 +if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then + echo "Saving image to S3" + docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz +fi diff --git a/multidim-interop/go/v0.22/Dockerfile b/multidim-interop/go/v0.22/Dockerfile index 7f4da95bb..1da43fff4 100644 --- a/multidim-interop/go/v0.22/Dockerfile +++ b/multidim-interop/go/v0.22/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM golang:1.19-alpine +FROM golang:1.19-alpine AS builder WORKDIR /app @@ -12,4 +12,7 @@ COPY *.go ./ RUN go build -o /testplan +FROM alpine:3.17 +COPY --from=builder /testplan /testplan + ENTRYPOINT [ "/testplan"] \ No newline at end of file diff --git a/multidim-interop/go/v0.23/Dockerfile b/multidim-interop/go/v0.23/Dockerfile index 7f4da95bb..1da43fff4 100644 --- a/multidim-interop/go/v0.23/Dockerfile +++ b/multidim-interop/go/v0.23/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM golang:1.19-alpine +FROM golang:1.19-alpine AS builder WORKDIR /app @@ -12,4 +12,7 @@ COPY *.go ./ RUN go build -o /testplan +FROM alpine:3.17 +COPY --from=builder /testplan /testplan + ENTRYPOINT [ "/testplan"] \ No newline at end of file diff --git a/multidim-interop/go/v0.24/Dockerfile b/multidim-interop/go/v0.24/Dockerfile index 7f4da95bb..1da43fff4 100644 --- a/multidim-interop/go/v0.24/Dockerfile +++ b/multidim-interop/go/v0.24/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM golang:1.19-alpine +FROM golang:1.19-alpine AS builder WORKDIR /app @@ -12,4 +12,7 @@ COPY *.go ./ RUN go build -o /testplan +FROM alpine:3.17 +COPY --from=builder /testplan /testplan + ENTRYPOINT [ "/testplan"] \ No newline at end of file From 93d4f712b282c738968828dd9a1a48e26bbfa702 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 31 Mar 2023 14:49:23 -0700 Subject: [PATCH 02/25] Pass in abort controller signal --- multidim-interop/dockerBuildWrapper.sh | 6 ++++-- multidim-interop/src/compose-runner.ts | 14 ++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/multidim-interop/dockerBuildWrapper.sh b/multidim-interop/dockerBuildWrapper.sh index 7d62c1a3f..c4b377a01 100755 --- a/multidim-interop/dockerBuildWrapper.sh +++ b/multidim-interop/dockerBuildWrapper.sh @@ -15,7 +15,7 @@ echo "Build hash is: $BUILD_HASH" # If we don't have this image, and we have an S3 bucket defined, try to load it from S3. # If we have this image, it's probably faster to rebuild the parts that have changed rather than fetch from S3. -if (! docker image inspect $IMAGE_NAME -f "{{.Id}}") && [[ -n "${AWS_BUCKET}" ]]; then +if (! docker image inspect $IMAGE_NAME -f "{{.Id}}") && [[ -n "${AWS_BUCKET:-}" ]]; then echo "Trying to load image from S3" aws s3 cp s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz - | docker image load && \ echo "Loaded image from S3" && exit 0 \ @@ -37,7 +37,9 @@ docker buildx build \ -t $IMAGE_NAME $CACHING_OPTIONS "$@" # If we're in CI, let's save the image to S3 -if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then +# Check if CI is set, and if we have an S3 bucket defined + +if [[ -n "${CI:-}" ]] && [[ -n "${AWS_BUCKET:-}" ]]; then echo "Saving image to S3" docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz fi diff --git a/multidim-interop/src/compose-runner.ts b/multidim-interop/src/compose-runner.ts index 2ad9982d0..099f9332a 100644 --- a/multidim-interop/src/compose-runner.ts +++ b/multidim-interop/src/compose-runner.ts @@ -8,6 +8,7 @@ import { ComposeSpecification, PropertiesServices } from "../compose-spec/compos import { stringify } from 'yaml'; const exec = util.promisify(execStd); +const timeoutSecs = 3 * 60 export type RunOpts = { up: { @@ -43,14 +44,11 @@ export async function run(namespace: string, compose: ComposeSpecification, opts } try { - const timeoutSecs = 3 * 60 - let timeoutId - const { stdout, stderr } = - (await Promise.race([ - exec(`docker compose -f ${path.join(dir, "compose.yaml")} up ${upFlags.join(" ")}`), - // Timeout - uses any type because this will only reject the promise. - new Promise((resolve, reject) => { timeoutId = setTimeout(() => reject("Timeout"), 1000 * timeoutSecs) }) - ])) + const timeoutId = setTimeout(() => controller.abort(), 1000 * timeoutSecs) + + const controller = new AbortController(); + const { signal } = controller; + const { stdout, stderr } = await exec(`docker compose -f ${path.join(dir, "compose.yaml")} up ${upFlags.join(" ")}`, { signal }) clearTimeout(timeoutId) const testResults = stdout.match(/.*dialer.*({.*)/) if (testResults === null || testResults.length < 2) { From b2d63503db1e37f25975e4731ada65e901b7bdb9 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 3 Apr 2023 11:34:57 -0700 Subject: [PATCH 03/25] Derive input hash from args, build_context, and dockerfile (even if dockerfile is outside of build_context) --- multidim-interop/dockerBuildWrapper.sh | 49 +++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/multidim-interop/dockerBuildWrapper.sh b/multidim-interop/dockerBuildWrapper.sh index c4b377a01..071a5a694 100755 --- a/multidim-interop/dockerBuildWrapper.sh +++ b/multidim-interop/dockerBuildWrapper.sh @@ -2,15 +2,56 @@ set -eou pipefail set -x -# Print the hash of our current working directory. +BUILD_CONTEXT="${@: -1}" +if [[ ! -d "$BUILD_CONTEXT" ]]; then + echo "Error: Last argument must be a directory" + exit 1 +fi + +REST_ARGS="${@:1:$#-1}" # All arguments except the last one + +# Print the hash of the build context. This is our input hash. # We'll use this to store the build cache in S3. -CWD_HASH=$(tar --sort=name \ +INPUT_HASH=$(tar --sort=name \ --mtime="1970-01-01 00:00Z" \ --owner=0 --group=0 \ + --directory $BUILD_CONTEXT \ -cvf - . | sha256sum | sed 's/.$//' | xargs) # Sed removes the trailing dash, xargs removes the spaces + +# Add the arguments to the hash, so we have a different cache key if args change. +INPUT_HASH=$(echo "$INPUT_HASH $REST_ARGS" | sha256sum | sed 's/.$//' | xargs) + ARCH=$(docker info -f "{{.Architecture}}") -BUILD_HASH="$IMAGE_NAME-$CWD_HASH-$ARCH" +# Get the dockerfile from the arguments, it's either -f or --file +DOCKERFILE=$(echo "$REST_ARGS" | grep -oE "(-f|--file) [^ ]*") || echo "" +if [[ -z "$DOCKERFILE" ]]; then + # If no dockerfile is specified, use the default + DOCKERFILE="Dockerfile" +else + DOCKERFILE=$(echo "$DOCKERFILE" | cut -d' ' -f2) +fi + +DOCKERFILE_PATH=$(readlink -f "$DOCKERFILE") +BUILD_CONTEXT_PATH=$(readlink -f $BUILD_CONTEXT) +if [[ -f "$DOCKERFILE_PATH" ]]; then + case $DOCKERFILE_PATH in + $BUILD_CONTEXT_PATH/*) + # Dockerfile is within the build context, so it's covered under the + # build context hash + ;; + *) + # Dockerfile is outside the build context. Let's add its contents to the hash + INPUT_HASH=$(cat $DOCKERFILE_PATH | cat - <(echo " # context_hash = $INPUT_HASH") | sha256sum | sed 's/.$//' | xargs) ;; + esac + + echo "Dockerfile is: $DOCKERFILE_PATH, cwd is: $BUILD_CONTEXT_PATH" +else + echo "Error: Dockerfile not found" + exit 1 +fi + +BUILD_HASH="$IMAGE_NAME-$INPUT_HASH-$ARCH" echo "Build hash is: $BUILD_HASH" # If we don't have this image, and we have an S3 bucket defined, try to load it from S3. @@ -34,7 +75,7 @@ CACHING_OPTIONS="" docker buildx build \ --load \ - -t $IMAGE_NAME $CACHING_OPTIONS "$@" + -t $IMAGE_NAME $CACHING_OPTIONS "$REST_ARGS" $BUILD_CONTEXT # If we're in CI, let's save the image to S3 # Check if CI is set, and if we have an S3 bucket defined From de1553d7d56f5cf6701bf7ca3016976baa14c1ce Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 3 Apr 2023 13:16:35 -0700 Subject: [PATCH 04/25] Fix restargs to build command --- multidim-interop/dockerBuildWrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidim-interop/dockerBuildWrapper.sh b/multidim-interop/dockerBuildWrapper.sh index 071a5a694..91e255a63 100755 --- a/multidim-interop/dockerBuildWrapper.sh +++ b/multidim-interop/dockerBuildWrapper.sh @@ -75,7 +75,7 @@ CACHING_OPTIONS="" docker buildx build \ --load \ - -t $IMAGE_NAME $CACHING_OPTIONS "$REST_ARGS" $BUILD_CONTEXT + -t $IMAGE_NAME $CACHING_OPTIONS $REST_ARGS $BUILD_CONTEXT # If we're in CI, let's save the image to S3 # Check if CI is set, and if we have an S3 bucket defined From 8b7bbabd75be3e91c07fb8cd04ae1425234f734c Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 10:16:04 -0700 Subject: [PATCH 05/25] Cache differently --- multidim-interop/Makefile | 1 - multidim-interop/dockerBuildWrapper.sh | 86 -------------------- multidim-interop/go/v0.22/Makefile | 23 +++++- multidim-interop/go/v0.23/Dockerfile | 2 +- multidim-interop/go/v0.23/Makefile | 23 +++++- multidim-interop/go/v0.24/Makefile | 23 +++++- multidim-interop/go/v0.25/Makefile | 20 ++++- multidim-interop/go/v0.26/Makefile | 21 ++++- multidim-interop/helpers/buildCacheKey.sh | 13 +++ multidim-interop/helpers/hashFiles.sh | 5 ++ multidim-interop/helpers/saveCache.sh | 9 ++ multidim-interop/helpers/shouldUseCache.sh | 18 ++++ multidim-interop/helpers/tryLoadCache.sh | 13 +++ multidim-interop/js/.gitignore | 1 + multidim-interop/js/v0.41/ChromiumDockerfile | 22 +---- multidim-interop/js/v0.41/Dockerfile | 8 +- multidim-interop/js/v0.41/Makefile | 19 ++--- multidim-interop/js/v0.41/build.ts | 42 ++++++++++ multidim-interop/js/v0.42/ChromiumDockerfile | 22 +---- multidim-interop/js/v0.42/Dockerfile | 2 +- multidim-interop/js/v0.42/Makefile | 19 ++--- multidim-interop/js/v0.42/build.ts | 42 ++++++++++ multidim-interop/js/v0.42/package.json | 2 +- multidim-interop/nim/v1.0/Makefile | 20 ++++- multidim-interop/rust/v0.48/Makefile | 19 ++++- multidim-interop/rust/v0.49/Makefile | 19 ++++- multidim-interop/rust/v0.50/Makefile | 20 ++++- multidim-interop/rust/v0.51/Makefile | 18 +++- 28 files changed, 348 insertions(+), 184 deletions(-) delete mode 100755 multidim-interop/dockerBuildWrapper.sh create mode 100755 multidim-interop/helpers/buildCacheKey.sh create mode 100755 multidim-interop/helpers/hashFiles.sh create mode 100755 multidim-interop/helpers/saveCache.sh create mode 100755 multidim-interop/helpers/shouldUseCache.sh create mode 100755 multidim-interop/helpers/tryLoadCache.sh create mode 100644 multidim-interop/js/.gitignore create mode 100755 multidim-interop/js/v0.41/build.ts create mode 100755 multidim-interop/js/v0.42/build.ts diff --git a/multidim-interop/Makefile b/multidim-interop/Makefile index 3e2b6d085..c0327cd08 100644 --- a/multidim-interop/Makefile +++ b/multidim-interop/Makefile @@ -13,5 +13,4 @@ $(RUST_SUBDIRS): $(NIM_SUBDIRS): $(MAKE) -C $@ - .PHONY: $(GO_SUBDIRS) $(JS_SUBDIRS) $(RUST_SUBDIRS) $(NIM_SUBDIRS) all diff --git a/multidim-interop/dockerBuildWrapper.sh b/multidim-interop/dockerBuildWrapper.sh deleted file mode 100755 index 91e255a63..000000000 --- a/multidim-interop/dockerBuildWrapper.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env /bin/bash -set -eou pipefail -set -x - -BUILD_CONTEXT="${@: -1}" -if [[ ! -d "$BUILD_CONTEXT" ]]; then - echo "Error: Last argument must be a directory" - exit 1 -fi - -REST_ARGS="${@:1:$#-1}" # All arguments except the last one - -# Print the hash of the build context. This is our input hash. -# We'll use this to store the build cache in S3. -INPUT_HASH=$(tar --sort=name \ - --mtime="1970-01-01 00:00Z" \ - --owner=0 --group=0 \ - --directory $BUILD_CONTEXT \ - -cvf - . | sha256sum | sed 's/.$//' | xargs) # Sed removes the trailing dash, xargs removes the spaces - -# Add the arguments to the hash, so we have a different cache key if args change. -INPUT_HASH=$(echo "$INPUT_HASH $REST_ARGS" | sha256sum | sed 's/.$//' | xargs) - -ARCH=$(docker info -f "{{.Architecture}}") - -# Get the dockerfile from the arguments, it's either -f or --file -DOCKERFILE=$(echo "$REST_ARGS" | grep -oE "(-f|--file) [^ ]*") || echo "" -if [[ -z "$DOCKERFILE" ]]; then - # If no dockerfile is specified, use the default - DOCKERFILE="Dockerfile" -else - DOCKERFILE=$(echo "$DOCKERFILE" | cut -d' ' -f2) -fi - -DOCKERFILE_PATH=$(readlink -f "$DOCKERFILE") -BUILD_CONTEXT_PATH=$(readlink -f $BUILD_CONTEXT) -if [[ -f "$DOCKERFILE_PATH" ]]; then - case $DOCKERFILE_PATH in - $BUILD_CONTEXT_PATH/*) - # Dockerfile is within the build context, so it's covered under the - # build context hash - ;; - *) - # Dockerfile is outside the build context. Let's add its contents to the hash - INPUT_HASH=$(cat $DOCKERFILE_PATH | cat - <(echo " # context_hash = $INPUT_HASH") | sha256sum | sed 's/.$//' | xargs) ;; - esac - - echo "Dockerfile is: $DOCKERFILE_PATH, cwd is: $BUILD_CONTEXT_PATH" -else - echo "Error: Dockerfile not found" - exit 1 -fi - -BUILD_HASH="$IMAGE_NAME-$INPUT_HASH-$ARCH" -echo "Build hash is: $BUILD_HASH" - -# If we don't have this image, and we have an S3 bucket defined, try to load it from S3. -# If we have this image, it's probably faster to rebuild the parts that have changed rather than fetch from S3. -if (! docker image inspect $IMAGE_NAME -f "{{.Id}}") && [[ -n "${AWS_BUCKET:-}" ]]; then - echo "Trying to load image from S3" - aws s3 cp s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz - | docker image load && \ - echo "Loaded image from S3" && exit 0 \ - || echo "Failed to load image from S3" -fi - -# TODO maybe get rid of this caching mode -CACHING_OPTIONS="" -# If in CI and we have a defined cache bucket, use caching -# if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then - # CACHING_OPTIONS="\ - # --cache-to type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME \ - # --cache-from type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME" -# fi -# echo "Caching options: $CI" - -docker buildx build \ - --load \ - -t $IMAGE_NAME $CACHING_OPTIONS $REST_ARGS $BUILD_CONTEXT - -# If we're in CI, let's save the image to S3 -# Check if CI is set, and if we have an S3 bucket defined - -if [[ -n "${CI:-}" ]] && [[ -n "${AWS_BUCKET:-}" ]]; then - echo "Saving image to S3" - docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_HASH.tar.gz -fi diff --git a/multidim-interop/go/v0.22/Makefile b/multidim-interop/go/v0.22/Makefile index 9f0d96c70..75b2a921e 100644 --- a/multidim-interop/go/v0.22/Makefile +++ b/multidim-interop/go/v0.22/Makefile @@ -1,11 +1,26 @@ -image_name := go-v0.22 +imageName := go-v0.22 +cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} + +all: image.json + +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: Dockerfile main.go go.mod go.sum - IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh . - docker image inspect ${image_name} -f "{{.Id}}" | \ + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif .PHONY: clean clean: - rm image.json \ No newline at end of file + rm image.json diff --git a/multidim-interop/go/v0.23/Dockerfile b/multidim-interop/go/v0.23/Dockerfile index 1da43fff4..62f43b0c6 100644 --- a/multidim-interop/go/v0.23/Dockerfile +++ b/multidim-interop/go/v0.23/Dockerfile @@ -15,4 +15,4 @@ RUN go build -o /testplan FROM alpine:3.17 COPY --from=builder /testplan /testplan -ENTRYPOINT [ "/testplan"] \ No newline at end of file +ENTRYPOINT [ "/testplan"] diff --git a/multidim-interop/go/v0.23/Makefile b/multidim-interop/go/v0.23/Makefile index fce12d5e7..2268d2dc4 100644 --- a/multidim-interop/go/v0.23/Makefile +++ b/multidim-interop/go/v0.23/Makefile @@ -1,11 +1,26 @@ -image_name := go-v0.23 +imageName := go-v0.23 +cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} + +all: image.json + +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: Dockerfile main.go go.mod go.sum - IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh . - docker image inspect ${image_name} -f "{{.Id}}" | \ + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif .PHONY: clean clean: - rm image.json \ No newline at end of file + rm image.json diff --git a/multidim-interop/go/v0.24/Makefile b/multidim-interop/go/v0.24/Makefile index f9b3736e3..4afbea93d 100644 --- a/multidim-interop/go/v0.24/Makefile +++ b/multidim-interop/go/v0.24/Makefile @@ -1,11 +1,26 @@ -image_name := go-v0.24 +imageName := go-v0.24 +cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} + +all: image.json + +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: Dockerfile main.go go.mod go.sum - IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh . - docker image inspect ${image_name} -f "{{.Id}}" | \ + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif .PHONY: clean clean: - rm image.json \ No newline at end of file + rm image.json diff --git a/multidim-interop/go/v0.25/Makefile b/multidim-interop/go/v0.25/Makefile index 6eb6526d1..3e9b3626c 100644 --- a/multidim-interop/go/v0.25/Makefile +++ b/multidim-interop/go/v0.25/Makefile @@ -1,12 +1,24 @@ -image_name := go-v0.25 +imageName := go-v0.25 commitSha := 5741b6c9bbcc1185bdf94d816dca966b37ce61ff +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: go-libp2p-${commitSha} - cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ + cd go-libp2p-${commitSha} && docker build -t ${imageName} -f test-plans/PingDockerfile . + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +endif go-libp2p-${commitSha}: go-libp2p-${commitSha}.zip unzip -o go-libp2p-${commitSha}.zip @@ -17,4 +29,4 @@ go-libp2p-${commitSha}.zip: clean: rm image.json rm go-libp2p-*.zip - rm -rf go-libp2p-* \ No newline at end of file + rm -rf go-libp2p-* diff --git a/multidim-interop/go/v0.26/Makefile b/multidim-interop/go/v0.26/Makefile index c14ed9357..2934394b6 100644 --- a/multidim-interop/go/v0.26/Makefile +++ b/multidim-interop/go/v0.26/Makefile @@ -1,12 +1,25 @@ -image_name := go-v0.26 +imageName := go-v0.26 commitSha := 59a14cf3194d5d057c45cb1dbc7b1af3a116bc7a +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: go-libp2p-${commitSha} - cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ + cd go-libp2p-${commitSha} && docker build -t ${imageName} -f test-plans/PingDockerfile . + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +endif go-libp2p-${commitSha}: go-libp2p-${commitSha}.zip unzip -o go-libp2p-${commitSha}.zip @@ -17,4 +30,4 @@ go-libp2p-${commitSha}.zip: clean: rm image.json rm go-libp2p-*.zip - rm -rf go-libp2p-* \ No newline at end of file + rm -rf go-libp2p-* diff --git a/multidim-interop/helpers/buildCacheKey.sh b/multidim-interop/helpers/buildCacheKey.sh new file mode 100755 index 000000000..8cdee9bf7 --- /dev/null +++ b/multidim-interop/helpers/buildCacheKey.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail +set -x + +# Assert that IMAGE_NAME is not empty +if [ -z "$IMAGE_NAME" ]; then + echo "IMAGE_NAME is not set" + exit 1 +fi + +ARCH=$(docker info -f "{{.Architecture}}") +BUILD_CACHE_KEY="$IMAGE_NAME-$CACHE_KEY-$ARCH" +echo $BUILD_CACHE_KEY diff --git a/multidim-interop/helpers/hashFiles.sh b/multidim-interop/helpers/hashFiles.sh new file mode 100755 index 000000000..c7533f09c --- /dev/null +++ b/multidim-interop/helpers/hashFiles.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail + +# sort all the files passed in by name, then hash them +echo "$@" | xargs -n1 | sort | xargs sha256sum | sha256sum | sed 's/.$//' | xargs diff --git a/multidim-interop/helpers/saveCache.sh b/multidim-interop/helpers/saveCache.sh new file mode 100755 index 000000000..ccbc53763 --- /dev/null +++ b/multidim-interop/helpers/saveCache.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail +set -x + +script_dir=$(dirname "$0") +BUILD_CACHE_KEY=$($script_dir/buildCacheKey.sh) +AWS_BUCKET=${AWS_BUCKET:-libp2p-by-tf-aws-bootstrap} + +docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz diff --git a/multidim-interop/helpers/shouldUseCache.sh b/multidim-interop/helpers/shouldUseCache.sh new file mode 100755 index 000000000..b16ac7c88 --- /dev/null +++ b/multidim-interop/helpers/shouldUseCache.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail +set -x + +# shouldUseCache.sh - Returns exit code 0 if we don't have the image locally and we have a cache hit. + +AWS_BUCKET=${AWS_BUCKET:-libp2p-by-tf-aws-bootstrap} + +script_dir=$(dirname "$0") +BUILD_CACHE_KEY=$($script_dir/buildCacheKey.sh) + +# If we already have this image name in docker, lets not use the cache, since a +# small change will probaby be faster than refetch the cache +if docker image inspect $IMAGE_NAME -f "{{.Id}}" &> /dev/null; then + exit 1; +fi + +curl --fail-with-body --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz diff --git a/multidim-interop/helpers/tryLoadCache.sh b/multidim-interop/helpers/tryLoadCache.sh new file mode 100755 index 000000000..c6ea4ea43 --- /dev/null +++ b/multidim-interop/helpers/tryLoadCache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail +set -x + +script_dir=$(dirname "$0") +BUILD_CACHE_KEY=$($script_dir/buildCacheKey.sh) +AWS_BUCKET=${AWS_BUCKET:-libp2p-by-tf-aws-bootstrap} + + +curl https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz \ + | docker image load && \ + exit 0 \ + || exit 1 diff --git a/multidim-interop/js/.gitignore b/multidim-interop/js/.gitignore new file mode 100644 index 000000000..d3d02c1fc --- /dev/null +++ b/multidim-interop/js/.gitignore @@ -0,0 +1 @@ +*-image.json diff --git a/multidim-interop/js/v0.41/ChromiumDockerfile b/multidim-interop/js/v0.41/ChromiumDockerfile index de8c51bd0..c522c85db 100644 --- a/multidim-interop/js/v0.41/ChromiumDockerfile +++ b/multidim-interop/js/v0.41/ChromiumDockerfile @@ -1,22 +1,6 @@ # syntax=docker/dockerfile:1 -FROM mcr.microsoft.com/playwright +ARG BASE_IMAGE +FROM $BASE_IMAGE -WORKDIR /app - - -COPY package*.json . - -RUN npm ci - -# Install browsers -RUN ./node_modules/.bin/playwright install - -COPY tsconfig.json . -COPY .aegir.js . -COPY test ./test -COPY src ./src - -RUN npm run build - -ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ] \ No newline at end of file +ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ] diff --git a/multidim-interop/js/v0.41/Dockerfile b/multidim-interop/js/v0.41/Dockerfile index cadf6681d..ce894e653 100644 --- a/multidim-interop/js/v0.41/Dockerfile +++ b/multidim-interop/js/v0.41/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1 -FROM node:18 +# Using playwright so that we have the same base across NodeJS + Browser tests +FROM mcr.microsoft.com/playwright WORKDIR /app @@ -7,6 +8,9 @@ COPY package*.json . RUN npm ci +# Install browsers, Needed for the browser tests, but we do it here so we have the same base +RUN ./node_modules/.bin/playwright install + COPY tsconfig.json . COPY .aegir.js . COPY test ./test @@ -14,4 +18,4 @@ COPY src ./src RUN npm run build -ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "node" ] \ No newline at end of file +ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "node" ] diff --git a/multidim-interop/js/v0.41/Makefile b/multidim-interop/js/v0.41/Makefile index a186017fc..7d782ee15 100644 --- a/multidim-interop/js/v0.41/Makefile +++ b/multidim-interop/js/v0.41/Makefile @@ -1,19 +1,18 @@ -image_name := js-v0.21 +imageName := js-v0.41 TEST_SOURCES := $(wildcard test/*.ts) -all: chromium-image.json node-image.json +all: node-${imageName}-image.json chromium-${imageName}-image.json -chromium-image.json: ChromiumDockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js - IMAGE_NAME=chromium-${image_name} ../../dockerBuildWrapper.sh -f ChromiumDockerfile . - docker image inspect chromium-${image_name} -f "{{.Id}}" | \ - xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ -node-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js - IMAGE_NAME=node-${image_name} ../../dockerBuildWrapper.sh -f Dockerfile . - docker image inspect node-${image_name} -f "{{.Id}}" | \ +node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js + PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} + +chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json + docker build -t chromium-${imageName} -f ChromiumDockerfile --build-arg="BASE_IMAGE=node-${imageName}" . + docker image inspect chromium-${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ .PHONY: clean clean: - rm *image.json \ No newline at end of file + rm *image.json diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts new file mode 100755 index 000000000..d9ab7ff3f --- /dev/null +++ b/multidim-interop/js/v0.41/build.ts @@ -0,0 +1,42 @@ +#! /usr/bin/env ts-node-esm + +import { execSync } from 'child_process'; +import * as fs from 'fs'; + +const cacheKey = execSync('git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); + +function shouldUseCache(imageName: string): boolean { + try { + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh`, { stdio: 'inherit' }); + return true; + } catch (e) { + return false; + } +} + +function buildImage(imageName: string, dockerfile: string) { + if (shouldUseCache(imageName)) { + console.log('Using cache'); + execSync(`CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh`, { stdio: 'inherit' }); + } else { + execSync(`docker build -t ${imageName} -f ${dockerfile} .`, { stdio: 'inherit' }); + + if (process.env.CI !== undefined && process.env.CI !== '') { + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh`, { stdio: 'inherit' }); + } + } + + const imageHash = execSync(`docker image inspect ${imageName} -f "{{.Id}}"`).toString().trim(); + fs.writeFileSync(`${imageName}-image.json`, JSON.stringify({ imageID: imageHash })); +} + +function main(imageName: string, dockerfile: string = 'Dockerfile') { + buildImage(imageName, dockerfile); +} + +const args = process.argv.slice(2); +const imageName = args[0]; +const dockerfile = args[1]; +main(imageName, dockerfile); + +export {} diff --git a/multidim-interop/js/v0.42/ChromiumDockerfile b/multidim-interop/js/v0.42/ChromiumDockerfile index de8c51bd0..c522c85db 100644 --- a/multidim-interop/js/v0.42/ChromiumDockerfile +++ b/multidim-interop/js/v0.42/ChromiumDockerfile @@ -1,22 +1,6 @@ # syntax=docker/dockerfile:1 -FROM mcr.microsoft.com/playwright +ARG BASE_IMAGE +FROM $BASE_IMAGE -WORKDIR /app - - -COPY package*.json . - -RUN npm ci - -# Install browsers -RUN ./node_modules/.bin/playwright install - -COPY tsconfig.json . -COPY .aegir.js . -COPY test ./test -COPY src ./src - -RUN npm run build - -ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ] \ No newline at end of file +ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ] diff --git a/multidim-interop/js/v0.42/Dockerfile b/multidim-interop/js/v0.42/Dockerfile index cadf6681d..24b8e1198 100644 --- a/multidim-interop/js/v0.42/Dockerfile +++ b/multidim-interop/js/v0.42/Dockerfile @@ -14,4 +14,4 @@ COPY src ./src RUN npm run build -ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "node" ] \ No newline at end of file +ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "node" ] diff --git a/multidim-interop/js/v0.42/Makefile b/multidim-interop/js/v0.42/Makefile index 3234688fd..df09f92af 100644 --- a/multidim-interop/js/v0.42/Makefile +++ b/multidim-interop/js/v0.42/Makefile @@ -1,19 +1,18 @@ -image_name := js-v0.42 +imageName := js-v0.42 TEST_SOURCES := $(wildcard test/*.ts) -all: chromium-image.json node-image.json +all: node-${imageName}-image.json chromium-${imageName}-image.json -chromium-image.json: ChromiumDockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js - IMAGE_NAME=chromium-${image_name} ../../dockerBuildWrapper.sh -f ChromiumDockerfile . - docker image inspect chromium-${image_name} -f "{{.Id}}" | \ - xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ -node-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js - IMAGE_NAME=node-${image_name} ../../dockerBuildWrapper.sh -f Dockerfile . - docker image inspect node-${image_name} -f "{{.Id}}" | \ +node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js + PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} + +chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json + docker build -t chromium-${imageName} -f ChromiumDockerfile --build-arg="BASE_IMAGE=node-${imageName}" . + docker image inspect chromium-${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ .PHONY: clean clean: - rm *image.json \ No newline at end of file + rm *image.json diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts new file mode 100755 index 000000000..d9ab7ff3f --- /dev/null +++ b/multidim-interop/js/v0.42/build.ts @@ -0,0 +1,42 @@ +#! /usr/bin/env ts-node-esm + +import { execSync } from 'child_process'; +import * as fs from 'fs'; + +const cacheKey = execSync('git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); + +function shouldUseCache(imageName: string): boolean { + try { + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh`, { stdio: 'inherit' }); + return true; + } catch (e) { + return false; + } +} + +function buildImage(imageName: string, dockerfile: string) { + if (shouldUseCache(imageName)) { + console.log('Using cache'); + execSync(`CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh`, { stdio: 'inherit' }); + } else { + execSync(`docker build -t ${imageName} -f ${dockerfile} .`, { stdio: 'inherit' }); + + if (process.env.CI !== undefined && process.env.CI !== '') { + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh`, { stdio: 'inherit' }); + } + } + + const imageHash = execSync(`docker image inspect ${imageName} -f "{{.Id}}"`).toString().trim(); + fs.writeFileSync(`${imageName}-image.json`, JSON.stringify({ imageID: imageHash })); +} + +function main(imageName: string, dockerfile: string = 'Dockerfile') { + buildImage(imageName, dockerfile); +} + +const args = process.argv.slice(2); +const imageName = args[0]; +const dockerfile = args[1]; +main(imageName, dockerfile); + +export {} diff --git a/multidim-interop/js/v0.42/package.json b/multidim-interop/js/v0.42/package.json index 936c7f8c2..bc873976f 100644 --- a/multidim-interop/js/v0.42/package.json +++ b/multidim-interop/js/v0.42/package.json @@ -37,4 +37,4 @@ "standard": "^17.0.0", "typescript": "^4.9.4" } -} \ No newline at end of file +} diff --git a/multidim-interop/nim/v1.0/Makefile b/multidim-interop/nim/v1.0/Makefile index 8a7891fa6..82221bafb 100644 --- a/multidim-interop/nim/v1.0/Makefile +++ b/multidim-interop/nim/v1.0/Makefile @@ -1,12 +1,26 @@ -image_name := nim-v1.0 +imageName := nim-v1.0 commitSha := 408dcf12bdf44dcd6f9021a6c795c472679d6d02 +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: main.nim nim-libp2p Dockerfile - IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh . - docker image inspect ${image_name} -f "{{.Id}}" | \ + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif + main.nim: ../mainv1.nim cp ../mainv1.nim main.nim diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index a461d1f60..5f0f9bc4f 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -1,12 +1,25 @@ image_name := rust-v0.48 commitSha := 7b3047d6d05d599f11f05938d4257e70de66ac12 +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json -image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else +image.json: main.nim nim-libp2p Dockerfile + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip unzip -o rust-libp2p-${commitSha}.zip diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index 9029e172b..7e1650c52 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -1,12 +1,25 @@ image_name := rust-v0.49 commitSha := 582c84043050eb0dd6e52d1bd0527175181d41cb +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json -image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else +image.json: main.nim nim-libp2p Dockerfile + docker build -t ${imageName} . + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip unzip -o rust-libp2p-${commitSha}.zip diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index 2cc4892f7..2efa03375 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -1,12 +1,26 @@ -image_name := rust-v0.50 +imageName := rust-v0.50 commitSha := beb66b5832384d9b813fcbf1f0fa01e6a64a9c5f +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ + cd rust-libp2p-${commitSha} && docker build -t ${imageName} -f interop-tests/Dockerfile . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif + rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip unzip -o rust-libp2p-${commitSha}.zip diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index 652370c79..6638f1022 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -1,12 +1,26 @@ image_name := rust-v0.51 commitSha := 1a9cf4f7760724032b729c43165716c7ecd842ad +cacheKey := ${commitSha} +shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json +ifneq (${shouldUseCache},) +# We have a cached image, let's use it +image.json: + echo "Using cached image ${shouldUseCache} f" + CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh + docker image inspect ${imageName} -f "{{.Id}}" | \ + xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ +else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f interop-tests/Dockerfile . - docker image inspect ${image_name} -f "{{.Id}}" | \ + cd rust-libp2p-${commitSha} && docker build -t ${imageName} -f interop-tests/Dockerfile . + docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ + # If we're in CI, save the cache + if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi +endif + rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip unzip -o rust-libp2p-${commitSha}.zip From 932b4960bd13f4b47b3fab9c5c29d66af83e8146 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 10:22:14 -0700 Subject: [PATCH 06/25] Silent check --- multidim-interop/helpers/shouldUseCache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidim-interop/helpers/shouldUseCache.sh b/multidim-interop/helpers/shouldUseCache.sh index b16ac7c88..fc8de4402 100755 --- a/multidim-interop/helpers/shouldUseCache.sh +++ b/multidim-interop/helpers/shouldUseCache.sh @@ -15,4 +15,4 @@ if docker image inspect $IMAGE_NAME -f "{{.Id}}" &> /dev/null; then exit 1; fi -curl --fail-with-body --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz +curl --fail-with-body --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz &> /dev/null From e0ecfef85a9029b49afdae09bd46a1a4240dbeb5 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 11:08:14 -0700 Subject: [PATCH 07/25] Use bash in makefile --- multidim-interop/go/v0.22/Makefile | 1 + multidim-interop/go/v0.23/Makefile | 1 + multidim-interop/go/v0.24/Makefile | 1 + multidim-interop/go/v0.25/Makefile | 1 + multidim-interop/go/v0.26/Makefile | 1 + multidim-interop/nim/v1.0/Makefile | 1 + multidim-interop/rust/v0.48/Makefile | 3 ++- multidim-interop/rust/v0.49/Makefile | 3 ++- multidim-interop/rust/v0.50/Makefile | 1 + multidim-interop/rust/v0.51/Makefile | 3 ++- 10 files changed, 13 insertions(+), 3 deletions(-) diff --git a/multidim-interop/go/v0.22/Makefile b/multidim-interop/go/v0.22/Makefile index 75b2a921e..1be262842 100644 --- a/multidim-interop/go/v0.22/Makefile +++ b/multidim-interop/go/v0.22/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := go-v0.22 cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} diff --git a/multidim-interop/go/v0.23/Makefile b/multidim-interop/go/v0.23/Makefile index 2268d2dc4..e0929a426 100644 --- a/multidim-interop/go/v0.23/Makefile +++ b/multidim-interop/go/v0.23/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := go-v0.23 cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} diff --git a/multidim-interop/go/v0.24/Makefile b/multidim-interop/go/v0.24/Makefile index 4afbea93d..1f84aa204 100644 --- a/multidim-interop/go/v0.24/Makefile +++ b/multidim-interop/go/v0.24/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := go-v0.24 cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} diff --git a/multidim-interop/go/v0.25/Makefile b/multidim-interop/go/v0.25/Makefile index 3e9b3626c..007225a6f 100644 --- a/multidim-interop/go/v0.25/Makefile +++ b/multidim-interop/go/v0.25/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := go-v0.25 commitSha := 5741b6c9bbcc1185bdf94d816dca966b37ce61ff cacheKey := ${commitSha} diff --git a/multidim-interop/go/v0.26/Makefile b/multidim-interop/go/v0.26/Makefile index 2934394b6..a267e868e 100644 --- a/multidim-interop/go/v0.26/Makefile +++ b/multidim-interop/go/v0.26/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := go-v0.26 commitSha := 59a14cf3194d5d057c45cb1dbc7b1af3a116bc7a cacheKey := ${commitSha} diff --git a/multidim-interop/nim/v1.0/Makefile b/multidim-interop/nim/v1.0/Makefile index 82221bafb..143b7d04f 100644 --- a/multidim-interop/nim/v1.0/Makefile +++ b/multidim-interop/nim/v1.0/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := nim-v1.0 commitSha := 408dcf12bdf44dcd6f9021a6c795c472679d6d02 cacheKey := ${commitSha} diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index 5f0f9bc4f..7d1d8bfcb 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -1,4 +1,5 @@ -image_name := rust-v0.48 +SHELL := /usr/bin/env bash +imageName := rust-v0.48 commitSha := 7b3047d6d05d599f11f05938d4257e70de66ac12 cacheKey := ${commitSha} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index 7e1650c52..e6d5f5e71 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -1,4 +1,5 @@ -image_name := rust-v0.49 +SHELL := /usr/bin/env bash +imageName := rust-v0.49 commitSha := 582c84043050eb0dd6e52d1bd0527175181d41cb cacheKey := ${commitSha} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index 2efa03375..9b7c8c185 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -1,3 +1,4 @@ +SHELL := /usr/bin/env bash imageName := rust-v0.50 commitSha := beb66b5832384d9b813fcbf1f0fa01e6a64a9c5f cacheKey := ${commitSha} diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index 6638f1022..b2ae15ddc 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -1,4 +1,5 @@ -image_name := rust-v0.51 +SHELL := /usr/bin/env bash +imageName := rust-v0.51 commitSha := 1a9cf4f7760724032b729c43165716c7ecd842ad cacheKey := ${commitSha} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} From a8fcce3dc3bf861187ed2474293f748539b8d2a3 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 11:40:44 -0700 Subject: [PATCH 08/25] Fix build.ts shebang --- multidim-interop/js/v0.41/build.ts | 2 +- multidim-interop/js/v0.42/build.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts index d9ab7ff3f..4b026536c 100755 --- a/multidim-interop/js/v0.41/build.ts +++ b/multidim-interop/js/v0.41/build.ts @@ -1,4 +1,4 @@ -#! /usr/bin/env ts-node-esm +#! /usr/bin/env ts-node-esm --skipProject -O {"module":"es2022"} import { execSync } from 'child_process'; import * as fs from 'fs'; diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts index d9ab7ff3f..4b026536c 100755 --- a/multidim-interop/js/v0.42/build.ts +++ b/multidim-interop/js/v0.42/build.ts @@ -1,4 +1,4 @@ -#! /usr/bin/env ts-node-esm +#! /usr/bin/env ts-node-esm --skipProject -O {"module":"es2022"} import { execSync } from 'child_process'; import * as fs from 'fs'; From ee27a530027e5c942cc14ee55a829261b8111d0c Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 11:58:10 -0700 Subject: [PATCH 09/25] Use -S in env --- multidim-interop/js/v0.41/build.ts | 2 +- multidim-interop/js/v0.42/build.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts index 4b026536c..9de470e5f 100755 --- a/multidim-interop/js/v0.41/build.ts +++ b/multidim-interop/js/v0.41/build.ts @@ -1,4 +1,4 @@ -#! /usr/bin/env ts-node-esm --skipProject -O {"module":"es2022"} +#! /usr/bin/env -S ts-node-esm --skipProject -O '{"module":"es2022"}' import { execSync } from 'child_process'; import * as fs from 'fs'; diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts index 4b026536c..9de470e5f 100755 --- a/multidim-interop/js/v0.42/build.ts +++ b/multidim-interop/js/v0.42/build.ts @@ -1,4 +1,4 @@ -#! /usr/bin/env ts-node-esm --skipProject -O {"module":"es2022"} +#! /usr/bin/env -S ts-node-esm --skipProject -O '{"module":"es2022"}' import { execSync } from 'child_process'; import * as fs from 'fs'; From cc1e261a17d244a2d6fa69c59ce3a6d06ead9830 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 12:18:40 -0700 Subject: [PATCH 10/25] Fix Dockerfile for use outside of buildx --- multidim-interop/js/v0.41/Dockerfile | 2 +- multidim-interop/js/v0.42/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multidim-interop/js/v0.41/Dockerfile b/multidim-interop/js/v0.41/Dockerfile index ce894e653..4c8d3a507 100644 --- a/multidim-interop/js/v0.41/Dockerfile +++ b/multidim-interop/js/v0.41/Dockerfile @@ -4,7 +4,7 @@ FROM mcr.microsoft.com/playwright WORKDIR /app -COPY package*.json . +COPY package*.json ./ RUN npm ci diff --git a/multidim-interop/js/v0.42/Dockerfile b/multidim-interop/js/v0.42/Dockerfile index 24b8e1198..b9bf62896 100644 --- a/multidim-interop/js/v0.42/Dockerfile +++ b/multidim-interop/js/v0.42/Dockerfile @@ -3,7 +3,7 @@ FROM node:18 WORKDIR /app -COPY package*.json . +COPY package*.json ./ RUN npm ci From a13f2aebb61fc3bdc3e0c8514b2e65bbb89b8b8e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 19:44:17 +0000 Subject: [PATCH 11/25] Use --fail in curl --- multidim-interop/helpers/shouldUseCache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidim-interop/helpers/shouldUseCache.sh b/multidim-interop/helpers/shouldUseCache.sh index fc8de4402..08922e09e 100755 --- a/multidim-interop/helpers/shouldUseCache.sh +++ b/multidim-interop/helpers/shouldUseCache.sh @@ -15,4 +15,4 @@ if docker image inspect $IMAGE_NAME -f "{{.Id}}" &> /dev/null; then exit 1; fi -curl --fail-with-body --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz &> /dev/null +curl --fail --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz &> /dev/null From 2293a30f17601c9cbb67cd721ad947e4600933a0 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 19:47:37 +0000 Subject: [PATCH 12/25] Fix typo --- multidim-interop/rust/v0.48/Makefile | 4 ++-- multidim-interop/rust/v0.49/Makefile | 4 ++-- multidim-interop/rust/v0.50/Makefile | 2 +- multidim-interop/rust/v0.51/Makefile | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index 7d1d8bfcb..b861573e6 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -14,8 +14,8 @@ image.json: docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else -image.json: main.nim nim-libp2p Dockerfile - docker build -t ${imageName} . +image.json: rust-libp2p-${commitSha} + cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index e6d5f5e71..4033dc173 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -14,8 +14,8 @@ image.json: docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else -image.json: main.nim nim-libp2p Dockerfile - docker build -t ${imageName} . +image.json: rust-libp2p-${commitSha} + cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index 9b7c8c185..2ff8d2f86 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -t ${imageName} -f interop-tests/Dockerfile . + cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index b2ae15ddc..d03277d61 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -t ${imageName} -f interop-tests/Dockerfile . + cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache From bf78222c9d76139f7b8be86f2903c559f3034a15 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 19:56:12 +0000 Subject: [PATCH 13/25] Echo something in shouldUseCache --- multidim-interop/helpers/shouldUseCache.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multidim-interop/helpers/shouldUseCache.sh b/multidim-interop/helpers/shouldUseCache.sh index 08922e09e..a4b28c2f0 100755 --- a/multidim-interop/helpers/shouldUseCache.sh +++ b/multidim-interop/helpers/shouldUseCache.sh @@ -16,3 +16,5 @@ if docker image inspect $IMAGE_NAME -f "{{.Id}}" &> /dev/null; then fi curl --fail --head https://s3.amazonaws.com/$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz &> /dev/null +# We need to echo something so that Make sees this as a target +echo $BUILD_CACHE_KEY From c4f1f39cfc344ff04c96b6a55dd34a6f4ee51712 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 20:07:43 +0000 Subject: [PATCH 14/25] Use buildx for rust builds --- multidim-interop/rust/v0.48/Makefile | 2 +- multidim-interop/rust/v0.49/Makefile | 2 +- multidim-interop/rust/v0.50/Makefile | 2 +- multidim-interop/rust/v0.51/Makefile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index b861573e6..143255c2b 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index 4033dc173..021490126 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index 2ff8d2f86..dd3bd18a0 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index d03277d61..42a1a0a65 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache From e9f44f38c00f152a0df6c6d8f9702df008bc0792 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 20:20:13 +0000 Subject: [PATCH 15/25] Load from buildx --- multidim-interop/rust/v0.48/Makefile | 2 +- multidim-interop/rust/v0.49/Makefile | 2 +- multidim-interop/rust/v0.50/Makefile | 2 +- multidim-interop/rust/v0.51/Makefile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index 143255c2b..de5758982 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build --load -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index 021490126..07de5b577 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build --load -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index dd3bd18a0..026879f74 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build --load -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index 42a1a0a65..8fc51cc32 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -15,7 +15,7 @@ image.json: xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ else image.json: rust-libp2p-${commitSha} - cd rust-libp2p-${commitSha} && docker buildx build -f interop-tests/Dockerfile -t ${imageName} . + cd rust-libp2p-${commitSha} && docker buildx build --load -f interop-tests/Dockerfile -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache From c0b6f1dc783908129247635f4a8f5dc15491f9ce Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 14:19:39 -0700 Subject: [PATCH 16/25] Fix version import --- multidim-interop/versions.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/multidim-interop/versions.ts b/multidim-interop/versions.ts index 1f1c4bae1..f516996b2 100644 --- a/multidim-interop/versions.ts +++ b/multidim-interop/versions.ts @@ -6,11 +6,11 @@ import rustv048 from "./rust/v0.48/image.json" import rustv049 from "./rust/v0.49/image.json" import rustv050 from "./rust/v0.50/image.json" import rustv051 from "./rust/v0.51/image.json" -import jsV041 from "./js/v0.41/node-image.json" -import jsV042 from "./js/v0.42/node-image.json" +import jsV041 from "./js/v0.41/node-js-v0.41-image.json" +import jsV042 from "./js/v0.42/node-js-v0.42-image.json" import nimv10 from "./nim/v1.0/image.json" -import chromiumJsV041 from "./js/v0.41/chromium-image.json" -import chromiumJsV042 from "./js/v0.42/chromium-image.json" +import chromiumJsV041 from "./js/v0.41/chromium-js-v0.41-image.json" +import chromiumJsV042 from "./js/v0.42/chromium-js-v0.42-image.json" export type Version = { id: string, From 160372489c435f8b16e9ff804e21351cfd48a57f Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 21:30:54 +0000 Subject: [PATCH 17/25] Start from playwright image --- multidim-interop/js/v0.42/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multidim-interop/js/v0.42/Dockerfile b/multidim-interop/js/v0.42/Dockerfile index b9bf62896..4c8d3a507 100644 --- a/multidim-interop/js/v0.42/Dockerfile +++ b/multidim-interop/js/v0.42/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1 -FROM node:18 +# Using playwright so that we have the same base across NodeJS + Browser tests +FROM mcr.microsoft.com/playwright WORKDIR /app @@ -7,6 +8,9 @@ COPY package*.json ./ RUN npm ci +# Install browsers, Needed for the browser tests, but we do it here so we have the same base +RUN ./node_modules/.bin/playwright install + COPY tsconfig.json . COPY .aegir.js . COPY test ./test From 3caee19f5bd41634fd3e1e018b892158cb548af5 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 4 Apr 2023 17:28:49 -0700 Subject: [PATCH 18/25] Try parallel builds --- .github/actions/run-interop-ping-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run-interop-ping-test/action.yml b/.github/actions/run-interop-ping-test/action.yml index ed99658e4..89475ffab 100644 --- a/.github/actions/run-interop-ping-test/action.yml +++ b/.github/actions/run-interop-ping-test/action.yml @@ -65,7 +65,7 @@ runs: - name: Build images working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }} - run: make + run: make -j 2 shell: bash - name: Run the test From 2564c1bfd044baeb748f71d1df34f313c5889f9d Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 5 Apr 2023 01:30:04 +0000 Subject: [PATCH 19/25] Normalized name for compose --- multidim-interop/src/generator.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/multidim-interop/src/generator.ts b/multidim-interop/src/generator.ts index 1d3986920..85163fe68 100644 --- a/multidim-interop/src/generator.ts +++ b/multidim-interop/src/generator.ts @@ -128,8 +128,10 @@ export async function buildTestSpecs(versions: Array): Promise Date: Wed, 5 Apr 2023 05:16:25 +0000 Subject: [PATCH 20/25] Move compose name sanitization to before serializing --- multidim-interop/src/compose-runner.ts | 8 +++++--- multidim-interop/src/generator.ts | 4 +--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/multidim-interop/src/compose-runner.ts b/multidim-interop/src/compose-runner.ts index 099f9332a..d82f28f96 100644 --- a/multidim-interop/src/compose-runner.ts +++ b/multidim-interop/src/compose-runner.ts @@ -32,8 +32,10 @@ export async function run(namespace: string, compose: ComposeSpecification, opts } await fs.mkdir(dir, { recursive: true }) - // Create compose.yaml file - await fs.writeFile(path.join(dir, "compose.yaml"), stringify(compose)) + // Create compose.yaml file. + // Some docker compose environments don't like the name field to have special characters + const sanitizedComposeName = compose?.name.replace(/[^a-zA-Z0-9_-]/g, "_") + await fs.writeFile(path.join(dir, "compose.yaml"), stringify({name: sanitizedComposeName, ...compose})) const upFlags: Array = [] if (opts.up.exitCodeFrom) { @@ -67,4 +69,4 @@ export async function run(namespace: string, compose: ComposeSpecification, opts } await fs.rm(dir, { recursive: true, force: true }) } -} \ No newline at end of file +} diff --git a/multidim-interop/src/generator.ts b/multidim-interop/src/generator.ts index 85163fe68..90b363cda 100644 --- a/multidim-interop/src/generator.ts +++ b/multidim-interop/src/generator.ts @@ -128,10 +128,8 @@ export async function buildTestSpecs(versions: Array): Promise Date: Wed, 5 Apr 2023 05:16:35 +0000 Subject: [PATCH 21/25] Double worker count --- .github/actions/run-interop-ping-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run-interop-ping-test/action.yml b/.github/actions/run-interop-ping-test/action.yml index 89475ffab..d3e17a7a2 100644 --- a/.github/actions/run-interop-ping-test/action.yml +++ b/.github/actions/run-interop-ping-test/action.yml @@ -70,7 +70,7 @@ runs: - name: Run the test working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }} - run: WORKER_COUNT=2 npm run test -- --extra-version=${{ inputs.extra-versions }} --name-filter=${{ inputs.test-filter }} + run: WORKER_COUNT=4 npm run test -- --extra-version=${{ inputs.extra-versions }} --name-filter=${{ inputs.test-filter }} shell: bash - name: Print the results From 1cd69352825aae6b7a5dea0ad2313c60751b7a17 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 5 Apr 2023 17:37:42 +0000 Subject: [PATCH 22/25] Use explicit PUSH_CACHE env var --- .github/actions/run-interop-ping-test/action.yml | 6 ++++++ multidim-interop/go/v0.22/Makefile | 3 +-- multidim-interop/go/v0.23/Makefile | 2 +- multidim-interop/go/v0.24/Makefile | 2 +- multidim-interop/go/v0.25/Makefile | 2 +- multidim-interop/go/v0.26/Makefile | 2 +- multidim-interop/helpers/maybePushCache.sh | 12 ++++++++++++ multidim-interop/helpers/saveCache.sh | 9 --------- multidim-interop/js/v0.41/build.ts | 5 +---- multidim-interop/js/v0.42/build.ts | 5 +---- multidim-interop/nim/v1.0/Makefile | 2 +- multidim-interop/rust/v0.48/Makefile | 2 +- multidim-interop/rust/v0.49/Makefile | 2 +- multidim-interop/rust/v0.50/Makefile | 2 +- multidim-interop/rust/v0.51/Makefile | 2 +- 15 files changed, 30 insertions(+), 28 deletions(-) create mode 100755 multidim-interop/helpers/maybePushCache.sh delete mode 100755 multidim-interop/helpers/saveCache.sh diff --git a/.github/actions/run-interop-ping-test/action.yml b/.github/actions/run-interop-ping-test/action.yml index d3e17a7a2..544afa9d1 100644 --- a/.github/actions/run-interop-ping-test/action.yml +++ b/.github/actions/run-interop-ping-test/action.yml @@ -34,6 +34,12 @@ runs: echo "AWS_REGION=${{ inputs.aws-region }}" >> $GITHUB_ENV shell: bash + - name: If we have a secret for s3, we should push to the cache. + if: inputs.s3-access-key-id != '' && inputs.s3-secret-access-key != '' + run: | + echo "PUSH_CACHE=true" >> $GITHUB_ENV + shell: bash + - name: Configure AWS credentials for S3 build cache if: inputs.s3-access-key-id != '' && inputs.s3-secret-access-key != '' uses: aws-actions/configure-aws-credentials@v1 diff --git a/multidim-interop/go/v0.22/Makefile b/multidim-interop/go/v0.22/Makefile index 1be262842..3bad1c36b 100644 --- a/multidim-interop/go/v0.22/Makefile +++ b/multidim-interop/go/v0.22/Makefile @@ -17,8 +17,7 @@ image.json: Dockerfile main.go go.mod go.sum docker build -t ${imageName} . docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ - # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif .PHONY: clean diff --git a/multidim-interop/go/v0.23/Makefile b/multidim-interop/go/v0.23/Makefile index e0929a426..2f187d5da 100644 --- a/multidim-interop/go/v0.23/Makefile +++ b/multidim-interop/go/v0.23/Makefile @@ -18,7 +18,7 @@ image.json: Dockerfile main.go go.mod go.sum docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif .PHONY: clean diff --git a/multidim-interop/go/v0.24/Makefile b/multidim-interop/go/v0.24/Makefile index 1f84aa204..7e99c3ea6 100644 --- a/multidim-interop/go/v0.24/Makefile +++ b/multidim-interop/go/v0.24/Makefile @@ -18,7 +18,7 @@ image.json: Dockerfile main.go go.mod go.sum docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif .PHONY: clean diff --git a/multidim-interop/go/v0.25/Makefile b/multidim-interop/go/v0.25/Makefile index 007225a6f..b3c3c1760 100644 --- a/multidim-interop/go/v0.25/Makefile +++ b/multidim-interop/go/v0.25/Makefile @@ -16,7 +16,7 @@ else image.json: go-libp2p-${commitSha} cd go-libp2p-${commitSha} && docker build -t ${imageName} -f test-plans/PingDockerfile . # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ endif diff --git a/multidim-interop/go/v0.26/Makefile b/multidim-interop/go/v0.26/Makefile index a267e868e..6f5de5ea9 100644 --- a/multidim-interop/go/v0.26/Makefile +++ b/multidim-interop/go/v0.26/Makefile @@ -17,7 +17,7 @@ else image.json: go-libp2p-${commitSha} cd go-libp2p-${commitSha} && docker build -t ${imageName} -f test-plans/PingDockerfile . # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ endif diff --git a/multidim-interop/helpers/maybePushCache.sh b/multidim-interop/helpers/maybePushCache.sh new file mode 100755 index 000000000..3596cc504 --- /dev/null +++ b/multidim-interop/helpers/maybePushCache.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env /bin/bash +set -eou pipefail +set -x + +if [[ -n "${PUSH_CACHE:-}" ]]; then + script_dir=$(dirname "$0") + BUILD_CACHE_KEY=$($script_dir/buildCacheKey.sh) + AWS_BUCKET=${AWS_BUCKET:-libp2p-by-tf-aws-bootstrap} + + docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz +fi + diff --git a/multidim-interop/helpers/saveCache.sh b/multidim-interop/helpers/saveCache.sh deleted file mode 100755 index ccbc53763..000000000 --- a/multidim-interop/helpers/saveCache.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env /bin/bash -set -eou pipefail -set -x - -script_dir=$(dirname "$0") -BUILD_CACHE_KEY=$($script_dir/buildCacheKey.sh) -AWS_BUCKET=${AWS_BUCKET:-libp2p-by-tf-aws-bootstrap} - -docker image save $IMAGE_NAME | gzip | aws s3 cp - s3://$AWS_BUCKET/imageCache/$BUILD_CACHE_KEY.tar.gz diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts index 9de470e5f..776e9ee9d 100755 --- a/multidim-interop/js/v0.41/build.ts +++ b/multidim-interop/js/v0.41/build.ts @@ -20,10 +20,7 @@ function buildImage(imageName: string, dockerfile: string) { execSync(`CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh`, { stdio: 'inherit' }); } else { execSync(`docker build -t ${imageName} -f ${dockerfile} .`, { stdio: 'inherit' }); - - if (process.env.CI !== undefined && process.env.CI !== '') { - execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh`, { stdio: 'inherit' }); - } + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh`, { stdio: 'inherit' }); } const imageHash = execSync(`docker image inspect ${imageName} -f "{{.Id}}"`).toString().trim(); diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts index 9de470e5f..776e9ee9d 100755 --- a/multidim-interop/js/v0.42/build.ts +++ b/multidim-interop/js/v0.42/build.ts @@ -20,10 +20,7 @@ function buildImage(imageName: string, dockerfile: string) { execSync(`CACHE_KEY=${cacheKey} IMAGE_NAME=${imageName} ../../helpers/tryLoadCache.sh`, { stdio: 'inherit' }); } else { execSync(`docker build -t ${imageName} -f ${dockerfile} .`, { stdio: 'inherit' }); - - if (process.env.CI !== undefined && process.env.CI !== '') { - execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh`, { stdio: 'inherit' }); - } + execSync(`IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh`, { stdio: 'inherit' }); } const imageHash = execSync(`docker image inspect ${imageName} -f "{{.Id}}"`).toString().trim(); diff --git a/multidim-interop/nim/v1.0/Makefile b/multidim-interop/nim/v1.0/Makefile index 143b7d04f..b86efcfa4 100644 --- a/multidim-interop/nim/v1.0/Makefile +++ b/multidim-interop/nim/v1.0/Makefile @@ -19,7 +19,7 @@ image.json: main.nim nim-libp2p Dockerfile docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif diff --git a/multidim-interop/rust/v0.48/Makefile b/multidim-interop/rust/v0.48/Makefile index de5758982..f765c731b 100644 --- a/multidim-interop/rust/v0.48/Makefile +++ b/multidim-interop/rust/v0.48/Makefile @@ -19,7 +19,7 @@ image.json: rust-libp2p-${commitSha} docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip diff --git a/multidim-interop/rust/v0.49/Makefile b/multidim-interop/rust/v0.49/Makefile index 07de5b577..69af99f4a 100644 --- a/multidim-interop/rust/v0.49/Makefile +++ b/multidim-interop/rust/v0.49/Makefile @@ -19,7 +19,7 @@ image.json: rust-libp2p-${commitSha} docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip diff --git a/multidim-interop/rust/v0.50/Makefile b/multidim-interop/rust/v0.50/Makefile index 026879f74..5a15ed297 100644 --- a/multidim-interop/rust/v0.50/Makefile +++ b/multidim-interop/rust/v0.50/Makefile @@ -19,7 +19,7 @@ image.json: rust-libp2p-${commitSha} docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif diff --git a/multidim-interop/rust/v0.51/Makefile b/multidim-interop/rust/v0.51/Makefile index 8fc51cc32..b80058078 100644 --- a/multidim-interop/rust/v0.51/Makefile +++ b/multidim-interop/rust/v0.51/Makefile @@ -19,7 +19,7 @@ image.json: rust-libp2p-${commitSha} docker image inspect ${imageName} -f "{{.Id}}" | \ xargs -I {} echo "{\"imageID\": \"{}\"}" > $@ # If we're in CI, save the cache - if [[ -n "$${CI:-}" ]]; then IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/saveCache.sh; fi + IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/maybePushCache.sh endif From 5255cea996afecd505acb4e42093e61864150634 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 7 Apr 2023 11:23:12 -0700 Subject: [PATCH 23/25] Fix name --- multidim-interop/src/compose-runner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidim-interop/src/compose-runner.ts b/multidim-interop/src/compose-runner.ts index d82f28f96..79b33515a 100644 --- a/multidim-interop/src/compose-runner.ts +++ b/multidim-interop/src/compose-runner.ts @@ -35,7 +35,7 @@ export async function run(namespace: string, compose: ComposeSpecification, opts // Create compose.yaml file. // Some docker compose environments don't like the name field to have special characters const sanitizedComposeName = compose?.name.replace(/[^a-zA-Z0-9_-]/g, "_") - await fs.writeFile(path.join(dir, "compose.yaml"), stringify({name: sanitizedComposeName, ...compose})) + await fs.writeFile(path.join(dir, "compose.yaml"), stringify({...compose, name: sanitizedComposeName})) const upFlags: Array = [] if (opts.up.exitCodeFrom) { From 4e9a600fcf59b598d9ab5d0dbceacfbc57fb94d8 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 7 Apr 2023 11:40:05 -0700 Subject: [PATCH 24/25] Don't use git ls-files since composite actions aren't in a git repo --- multidim-interop/go/v0.22/Makefile | 2 +- multidim-interop/go/v0.23/Makefile | 2 +- multidim-interop/go/v0.24/Makefile | 2 +- multidim-interop/js/v0.41/Makefile | 2 +- multidim-interop/js/v0.41/build.ts | 4 ++-- multidim-interop/js/v0.42/Makefile | 2 +- multidim-interop/js/v0.42/build.ts | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/multidim-interop/go/v0.22/Makefile b/multidim-interop/go/v0.22/Makefile index 3bad1c36b..7f2e637e0 100644 --- a/multidim-interop/go/v0.22/Makefile +++ b/multidim-interop/go/v0.22/Makefile @@ -1,7 +1,7 @@ SHELL := /usr/bin/env bash imageName := go-v0.22 -cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +cacheKey=${shell find . -type f | sort | grep -v image.json | tr "\n" " " | xargs ../../helpers/hashFiles.sh} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json diff --git a/multidim-interop/go/v0.23/Makefile b/multidim-interop/go/v0.23/Makefile index 2f187d5da..792ab5900 100644 --- a/multidim-interop/go/v0.23/Makefile +++ b/multidim-interop/go/v0.23/Makefile @@ -1,7 +1,7 @@ SHELL := /usr/bin/env bash imageName := go-v0.23 -cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +cacheKey=${shell find . -type f | sort | grep -v image.json | tr "\n" " " | xargs ../../helpers/hashFiles.sh} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json diff --git a/multidim-interop/go/v0.24/Makefile b/multidim-interop/go/v0.24/Makefile index 7e99c3ea6..e2a9052d2 100644 --- a/multidim-interop/go/v0.24/Makefile +++ b/multidim-interop/go/v0.24/Makefile @@ -1,7 +1,7 @@ SHELL := /usr/bin/env bash imageName := go-v0.24 -cacheKey=${shell git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh} +cacheKey=${shell find . -type f | sort | grep -v image.json | tr "\n" " " | xargs ../../helpers/hashFiles.sh} shouldUseCache := ${shell IMAGE_NAME=${imageName} CACHE_KEY=${cacheKey} ../../helpers/shouldUseCache.sh} all: image.json diff --git a/multidim-interop/js/v0.41/Makefile b/multidim-interop/js/v0.41/Makefile index 7d782ee15..1aa834701 100644 --- a/multidim-interop/js/v0.41/Makefile +++ b/multidim-interop/js/v0.41/Makefile @@ -4,7 +4,7 @@ TEST_SOURCES := $(wildcard test/*.ts) all: node-${imageName}-image.json chromium-${imageName}-image.json -node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js +node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js build.ts PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts index 776e9ee9d..074212f9d 100755 --- a/multidim-interop/js/v0.41/build.ts +++ b/multidim-interop/js/v0.41/build.ts @@ -1,9 +1,9 @@ -#! /usr/bin/env -S ts-node-esm --skipProject -O '{"module":"es2022"}' +#! /usr/bin/env -S ts-node-esm --skipProject -O {"module":"es2022"} import { execSync } from 'child_process'; import * as fs from 'fs'; -const cacheKey = execSync('git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); +const cacheKey = execSync('find . -type f | sort | grep -v image.json | grep -v ./dist/ | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); function shouldUseCache(imageName: string): boolean { try { diff --git a/multidim-interop/js/v0.42/Makefile b/multidim-interop/js/v0.42/Makefile index df09f92af..88fd1959e 100644 --- a/multidim-interop/js/v0.42/Makefile +++ b/multidim-interop/js/v0.42/Makefile @@ -4,7 +4,7 @@ TEST_SOURCES := $(wildcard test/*.ts) all: node-${imageName}-image.json chromium-${imageName}-image.json -node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js +node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js build.ts PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts index 776e9ee9d..8e260705e 100755 --- a/multidim-interop/js/v0.42/build.ts +++ b/multidim-interop/js/v0.42/build.ts @@ -3,7 +3,7 @@ import { execSync } from 'child_process'; import * as fs from 'fs'; -const cacheKey = execSync('git ls-files | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); +const cacheKey = execSync('find . -type f | sort | grep -v image.json | grep -v ./dist/ | tr "\n" " " | xargs ../../helpers/hashFiles.sh').toString().trim(); function shouldUseCache(imageName: string): boolean { try { From e057f2ca1223ea6b0c31045b7a9a1da5090d4500 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 7 Apr 2023 11:55:15 -0700 Subject: [PATCH 25/25] Shebang is not consistent across platforms --- multidim-interop/js/v0.41/Makefile | 2 +- multidim-interop/js/v0.41/build.ts | 2 -- multidim-interop/js/v0.42/Makefile | 2 +- multidim-interop/js/v0.42/build.ts | 2 -- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/multidim-interop/js/v0.41/Makefile b/multidim-interop/js/v0.41/Makefile index 1aa834701..cfb871e4a 100644 --- a/multidim-interop/js/v0.41/Makefile +++ b/multidim-interop/js/v0.41/Makefile @@ -5,7 +5,7 @@ all: node-${imageName}-image.json chromium-${imageName}-image.json node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js build.ts - PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} + PATH=$$PATH:../../node_modules/.bin ts-node-esm --skipProject -O '{"module":"es2022"}' ./build.ts node-${imageName} chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json docker build -t chromium-${imageName} -f ChromiumDockerfile --build-arg="BASE_IMAGE=node-${imageName}" . diff --git a/multidim-interop/js/v0.41/build.ts b/multidim-interop/js/v0.41/build.ts index 074212f9d..e83ef70b1 100755 --- a/multidim-interop/js/v0.41/build.ts +++ b/multidim-interop/js/v0.41/build.ts @@ -1,5 +1,3 @@ -#! /usr/bin/env -S ts-node-esm --skipProject -O {"module":"es2022"} - import { execSync } from 'child_process'; import * as fs from 'fs'; diff --git a/multidim-interop/js/v0.42/Makefile b/multidim-interop/js/v0.42/Makefile index 88fd1959e..e2a1b9fc6 100644 --- a/multidim-interop/js/v0.42/Makefile +++ b/multidim-interop/js/v0.42/Makefile @@ -5,7 +5,7 @@ all: node-${imageName}-image.json chromium-${imageName}-image.json node-${imageName}-image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js build.ts - PATH=$$PATH:../../node_modules/.bin ./build.ts node-${imageName} + PATH=$$PATH:../../node_modules/.bin ts-node-esm --skipProject -O '{"module":"es2022"}' ./build.ts node-${imageName} chromium-${imageName}-image.json: ChromiumDockerfile $(TEST_SOURCES) node-${imageName}-image.json docker build -t chromium-${imageName} -f ChromiumDockerfile --build-arg="BASE_IMAGE=node-${imageName}" . diff --git a/multidim-interop/js/v0.42/build.ts b/multidim-interop/js/v0.42/build.ts index 8e260705e..e83ef70b1 100755 --- a/multidim-interop/js/v0.42/build.ts +++ b/multidim-interop/js/v0.42/build.ts @@ -1,5 +1,3 @@ -#! /usr/bin/env -S ts-node-esm --skipProject -O '{"module":"es2022"}' - import { execSync } from 'child_process'; import * as fs from 'fs';