Cache images slightly differently (#165)

.github/actions/run-interop-ping-test/action.yml

@@ -34,6 +34,12 @@ runs:
         echo "AWS_REGION=${{ inputs.aws-region }}" >> $GITHUB_ENV
       shell: bash
 
+    - name: Configure AWS credentials for S3 build cache
+      if: inputs.s3-access-key-id != '' && inputs.s3-secret-access-key != ''
+      run: |
+        echo "PUSH_CACHE=true" >> $GITHUB_ENV
+      shell: bash
+
     - name: Configure AWS credentials for S3 build cache
       if: inputs.s3-access-key-id != '' && inputs.s3-secret-access-key != ''
       uses: aws-actions/configure-aws-credentials@v1
@@ -63,9 +69,15 @@ runs:
       run: npm ci
       shell: bash
 
-    - name: Build images
+    - name: Load cache and build
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: npm run cache -- load
+      shell: bash
+
+    - name: Push the image cache
+      if: env.PUSH_CACHE == 'true'
       working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
-      run: make
+      run: npm run cache -- push
       shell: bash
 
     - name: Run the test

.gitignore

@@ -42,4 +42,7 @@ __pycache__/
 
 ### NodeJS
 
-node_modules
+node_modules
+
+# ignore system files
+.DS_Store

multidim-interop/Makefile

@@ -1,7 +1,7 @@
-GO_SUBDIRS := $(wildcard go/*/.)
-JS_SUBDIRS := $(wildcard js/*/.)
-RUST_SUBDIRS := $(wildcard rust/*/.)
-NIM_SUBDIRS := $(wildcard nim/*/.)
+GO_SUBDIRS := $(wildcard impl/go/*/.)
+JS_SUBDIRS := $(wildcard impl/js/*/.)
+RUST_SUBDIRS := $(wildcard impl/rust/*/.)
+NIM_SUBDIRS := $(wildcard impl/nim/*/.)
 
 all: $(GO_SUBDIRS) $(JS_SUBDIRS) $(RUST_SUBDIRS) $(NIM_SUBDIRS)
 $(JS_SUBDIRS):

multidim-interop/README.md

@@ -1,11 +1,16 @@
 # Interoperability test
 
 This tests that different libp2p implementations can communicate with each other
-on each of their supported capabilites.
+on each of their supported capabilities.
 
 Each version of libp2p is defined in `versions.ts`. There the version defines
 its capabilities along with the id of its container image.
 
+This repo and tests adhere to these constraints:
+1. Be reproducible for a given commit.
+2. Caching is an optimization. Things should be fine without it.
+3. If we have a cache hit, be fast.
+
 # Test spec
 
 The implementation is run in a container and is passed parameters via
@@ -55,4 +60,19 @@ The listener should emit all diagnostic logs to `stderr`.
    process when the dialer finishes.
 5. If the timeout is hit, exit with a non-zero error code.
 
-On error, the listener should return a non-zero exit code.
+On error, the listener should return a non-zero exit code.
+
+# Caching
+
+The caching strategy is opinionated in an attempt to make things simpler and
+faster. Here's how it works:
+
+1. We cache the result of image.json in each implementation folder.
+2. The cache key is derived from the hashes of the files in the implementation folder.
+3. When loading from cache, if we have a cache hit, we load the image into
+   docker and create the image.json file. We then call `make -o image.json` to
+   allow the implementation to build any extra things from cache (e.g. JS-libp2p
+   builds browser images from the same base as node). If we have a cache miss,
+   we simply call `make` and build from scratch.
+4. When we push the cache we use the cache-key along with the docker platform
+   (arm64 vs x86_64).

multidim-interop/helpers/cache.ts

@@ -0,0 +1,148 @@
+const AWS_BUCKET = process.env.AWS_BUCKET || 'libp2p-by-tf-aws-bootstrap';
+const scriptDir = __dirname;
+
+import * as crypto from 'crypto';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as child_process from 'child_process';
+import ignore, { Ignore } from 'ignore'
+
+const multidimInteropDir = path.join(scriptDir, '..')
+const arch = child_process.execSync('docker info -f "{{.Architecture}}"').toString().trim();
+
+enum Mode {
+    LoadCache = 1,
+    PushCache,
+}
+const modeStr = process.argv[2];
+let mode: Mode
+switch (modeStr) {
+    case "push":
+        mode = Mode.PushCache
+        break
+    case "load":
+        mode = Mode.LoadCache
+        break
+    default:
+        throw new Error(`Unknown mode: ${modeStr}`)
+}
+
+(async () => {
+    for (const implFamily of fs.readdirSync(path.join(multidimInteropDir, 'impl'))) {
+        const ig = ignore()
+
+        addGitignoreIfPresent(ig, path.join(multidimInteropDir, ".gitignore"))
+        addGitignoreIfPresent(ig, path.join(multidimInteropDir, "..", ".gitignore"))
+
+        const implFamilyDir = path.join(multidimInteropDir, 'impl', implFamily)
+        addGitignoreIfPresent(ig, path.join(implFamilyDir, ".gitignore"))
+
+        for (const impl of fs.readdirSync(implFamilyDir)) {
+            const implFolder = fs.realpathSync(path.join(implFamilyDir, impl));
+            if (!fs.statSync(implFolder).isDirectory()) {
+                continue
+            }
+
+            addGitignoreIfPresent(ig, path.join(implFolder, ".gitignore"))
+
+            // Get all the files in the implFolder:
+            let files = walkDir(implFolder)
+            // Turn them into relative paths:
+            files = files.map(f => f.replace(implFolder + "/", ""))
+            // Ignore files that are in the .gitignore:
+            files = files.filter(ig.createFilter())
+            // Sort them to be deterministic
+            files = files.sort()
+
+            console.log(implFolder)
+            console.log("Files:", files)
+
+            // Turn them back into absolute paths:
+            files = files.map(f => path.join(implFolder, f))
+            const cacheKey = await hashFiles(files)
+            console.log("Cache key:", cacheKey)
+
+            if (mode == Mode.PushCache) {
+                console.log("Pushing cache")
+                try {
+                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
+                    if (res.ok) {
+                        console.log("Cache already exists")
+                    } else {
+                        // Read image id from image.json
+                        const imageID = JSON.parse(fs.readFileSync(path.join(implFolder, 'image.json')).toString()).imageID;
+                        console.log(`Pushing cache for ${impl}: ${imageID}`)
+                        child_process.execSync(`docker image save ${imageID} | gzip | aws s3 cp - s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`);
+                    }
+                } catch (e) {
+                    console.log("Failed to push image cache:", e)
+                }
+            } else if (mode == Mode.LoadCache) {
+                if (fs.existsSync(path.join(implFolder, 'image.json'))) {
+                    console.log("Already built")
+                    continue
+                }
+                console.log("Loading cache")
+                let cacheHit = false
+                try {
+                    // Check if the cache exists
+                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
+                    if (res.ok) {
+                        const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
+                        const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+                        if (loadedImageId) {
+                            console.log(`Cache hit for ${loadedImageId}`);
+                            fs.writeFileSync(path.join(implFolder, 'image.json'), JSON.stringify({ imageID: loadedImageId }) + "\n");
+                            cacheHit = true
+                        }
+                    } else {
+                        console.log("Cache not found")
+                    }
+                } catch (e) {
+                    console.log("Cache not found:", e)
+                }
+
+                if (cacheHit) {
+                    console.log("Building any remaining things from image.json")
+                    // We're building using -o image.json. This tells make to
+                    // not bother building image.json or anything it depends on.
+                    child_process.execSync(`make -o image.json`, { cwd: implFolder })
+                } else {
+                    console.log("No cache, building from scratch")
+                    child_process.execSync(`make`, { cwd: implFolder })
+                }
+            }
+        }
+    }
+})()
+
+function walkDir(dir: string) {
+    let results = [];
+    fs.readdirSync(dir).forEach(f => {
+        let dirPath = path.join(dir, f);
+        let isDirectory = fs.statSync(dirPath).isDirectory();
+        results = isDirectory ? results.concat(walkDir(dirPath)) : results.concat(path.join(dir, f));
+    });
+    return results;
+};
+
+async function hashFiles(files: string[]): Promise<string> {
+    const fileHashes = await Promise.all(
+        files.map(async (file) => {
+            const data = await fs.promises.readFile(file);
+            return crypto.createHash('sha256').update(data).digest('hex');
+        })
+    );
+    return crypto.createHash('sha256').update(fileHashes.join('')).digest('hex');
+}
+
+function addGitignoreIfPresent(ig: Ignore, pathStr: string): boolean {
+    try {
+        if (fs.statSync(pathStr).isFile()) {
+            ig.add(fs.readFileSync(pathStr).toString())
+        }
+        return true
+    } catch {
+        return false
+    }
+}

multidim-interop/go/v0.22/Makefile → multidim-interop/impl/go/v0.22/Makefile

@@ -1,11 +1,11 @@
 image_name := go-v0.22
 
 image.json: Dockerfile main.go go.mod go.sum
-	IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh .
+	IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
 .PHONY: clean
 
 clean:
-	rm image.json
+	rm image.json

multidim-interop/go/v0.23/Makefile → multidim-interop/impl/go/v0.23/Makefile

@@ -1,11 +1,11 @@
 image_name := go-v0.23
 
 image.json: Dockerfile main.go go.mod go.sum
-	IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh .
+	IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
 .PHONY: clean
 
 clean:
-	rm image.json
+	rm image.json

multidim-interop/go/v0.24/Makefile → multidim-interop/impl/go/v0.24/Makefile

@@ -1,11 +1,11 @@
 image_name := go-v0.24
 
 image.json: Dockerfile main.go go.mod go.sum
-	IMAGE_NAME=${image_name} ../../dockerBuildWrapper.sh .
+	IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
 .PHONY: clean
 
 clean:
-	rm image.json
+	rm image.json

multidim-interop/go/v0.25/Makefile → multidim-interop/impl/go/v0.25/Makefile

@@ -4,7 +4,7 @@ commitSha := 5741b6c9bbcc1185bdf94d816dca966b37ce61ff
 all: image.json
 
 image.json: go-libp2p-${commitSha}
-	cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile .
+	cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
@@ -17,4 +17,4 @@ go-libp2p-${commitSha}.zip:
 clean:
 	rm image.json
 	rm go-libp2p-*.zip
-	rm -rf go-libp2p-*
+	rm -rf go-libp2p-*

multidim-interop/go/v0.26/Makefile → multidim-interop/impl/go/v0.26/Makefile

@@ -4,7 +4,7 @@ commitSha := 59a14cf3194d5d057c45cb1dbc7b1af3a116bc7a
 all: image.json
 
 image.json: go-libp2p-${commitSha}
-	cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile .
+	cd go-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f test-plans/PingDockerfile .
 	docker image inspect ${image_name} -f "{{.Id}}" | \
 		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
 
@@ -17,4 +17,4 @@ go-libp2p-${commitSha}.zip:
 clean:
 	rm image.json
 	rm go-libp2p-*.zip
-	rm -rf go-libp2p-*
+	rm -rf go-libp2p-*

multidim-interop/impl/js/.gitignore

@@ -0,0 +1 @@
+*image.json

multidim-interop/impl/js/v0.41/ChromiumDockerfile

@@ -0,0 +1,6 @@
+# syntax=docker/dockerfile:1
+
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ]

multidim-interop/js/v0.42/ChromiumDockerfile → multidim-interop/impl/js/v0.41/Dockerfile

@@ -1,15 +1,14 @@
 # syntax=docker/dockerfile:1
-
+# Using playwright so that we have the same base across NodeJS + Browser tests
 FROM mcr.microsoft.com/playwright
 
 WORKDIR /app
 
-
-COPY package*.json .
+COPY package*.json ./
 
 RUN npm ci
 
-# Install browsers
+# Install browsers, Needed for the browser tests, but we do it here so we have the same base
 RUN ./node_modules/.bin/playwright install
 
 COPY tsconfig.json .
@@ -19,4 +18,4 @@ COPY src ./src
 
 RUN npm run build
 
-ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ]
+ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "node" ]

multidim-interop/impl/js/v0.41/Makefile

@@ -0,0 +1,23 @@
+image_name := js-v0.41
+TEST_SOURCES := $(wildcard test/*.ts)
+
+all: chromium-image.json node-image.json
+
+chromium-image.json: node-image.json
+	docker build -t chromium-${image_name} -f ChromiumDockerfile --build-arg="BASE_IMAGE=node-${image_name}" .
+	docker image inspect chromium-${image_name} -f "{{.Id}}" | \
+		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
+
+node-image.json: image.json
+	docker image tag $$(cat image.json | jq -r '.imageID') node-${image_name}
+	cp image.json node-image.json
+
+image.json: Dockerfile $(TEST_SOURCES) package.json package-lock.json .aegir.js
+	IMAGE_NAME=node-${image_name} ../../../dockerBuildWrapper.sh -f Dockerfile .
+	docker image inspect node-${image_name} -f "{{.Id}}" | \
+		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
+
+.PHONY: clean
+
+clean:
+	rm *image.json

multidim-interop/impl/js/v0.42/ChromiumDockerfile

@@ -0,0 +1,6 @@
+# syntax=docker/dockerfile:1
+
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+ENTRYPOINT [ "npm", "test", "--", "--build", "false", "--types", "false", "-t", "browser" ]