Fixes docker.io normal case rate-limiting (#171)

Each pull request implies many docker.io requests, this has led to
builds failing on account of rate limiting. This avoids that by using
the GitHub Container Registry (ghcr.io) instead of Docker Hub
(docker.io). ghcr.io is not rate-limited. Specifically, we use internal
copies of images used as parents or in testing. Since these aren't what
end users use anyway, this doesn't impact them.

At the end of the day, ghcr.io/tetratelabs/getenvoy-internal has tags
corresponding to our needs:

```yaml
          - source: busybox:1.32.1  # test image: ci/e2e/darwin/install_docker.sh
            target_tag: busybox
          - source: registry:2  # test image: docker-compose.yml
            target_tag: registry
          - source: rust:1.51.0  # parent image: images/extension-builders/rust/Dockerfile
            target_tag: rust
          - source: tinygo/tinygo:0.17.0  # parent image: images/extension-builders/tinygo/Dockerfile
            target_tag: tinygo
```

Updating our base layers or otherwise implies a change to
`.github/workflows/internal-images.yml` and re-triggering the workflow
or waiting until the next day for automatic publishing to occur.

Another change in this PR is removing the docker dependency from unit
tests via a mock. Even if we have safe images, we still shouldn't
subject unit tests to flakiness, extra time, or network dependencies
unless there is no other way.

.github/workflows/commit.yaml

@@ -35,13 +35,6 @@ jobs:
         with:
           go-version: '1.16.2'
 
-      - name: "Install 'Docker for Mac' (Latest)"
-        uses: docker-practice/actions-setup-docker@v1  # needed while TestGetEnvoyExtensionPush uses a real registry
-        if: runner.os == 'macOS'
-        with:
-          docker_buildx: false  # Install is flakey. When it, we can install it via docker/setup-buildx-action@v1
-        timeout-minutes: 20  # fail fast if MacOS install takes too long
-
       - name: "Verify clean check-in"
         run: make check
 
@@ -138,7 +131,7 @@ jobs:
 
       - name: "Build language-specific Docker build images"
         run: make builders
-        timeout-minutes: 10  # fail fast if MacOS runner becomes too slow
+        timeout-minutes: 20  # NOTE: the rust image is very large and can alone take 7 minutes to download and build
 
       - name: "Run e2e tests using the `getenvoy` binary built by the upstream job"
         # chmod to restore permissions lost in actions/download-artifact@v2

.github/workflows/internal-images.yml

@@ -0,0 +1,46 @@
+# yamllint --format github .github/workflows/internal-images.yml
+---
+name: internal-images
+
+# Refresh the tags once a day
+on:
+  schedule:
+    - cron: "23 3 * * *"
+  workflow_dispatch:  # Allows manual refresh
+
+# This copies images from docker.io to ghcr.io/tetratelabs/getenvoy-internal:$tag
+# Using these in tests and as a parent (FROM) avoids docker.io rate-limits particularly on pull requests.
+jobs:
+  copy-images:
+    strategy:
+      matrix:
+        # Be precise in tag versions to improve reproducibility
+        include:
+          - source: busybox:1.32.1  # test image: ci/e2e/darwin/install_docker.sh
+            target_tag: busybox
+          - source: registry:2  # test image: docker-compose.yml
+            target_tag: registry
+          - source: rust:1.51.0  # parent image: images/extension-builders/rust/Dockerfile
+            target_tag: rust
+          - source: tinygo/tinygo:0.17.0  # parent image: images/extension-builders/tinygo/Dockerfile
+            target_tag: tinygo
+    runs-on: ubuntu-latest
+    steps:
+      # Same as doing this locally: echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_TOKEN}" --password-stdin
+      - name: Login into GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          # GHCR_TOKEN=<hex token value>
+          #   - pushes Docker images to ghcr.io
+          #   - create via https://github.com/settings/tokens
+          #   - assign via https://github.com/organizations/tetratelabs/settings/secrets/actions
+          #   - needs repo:status, public_repo, write:packages, delete:packages
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Pull and push
+        run: |  # This will only push a single architecture, which is fine as we currently only support amd64
+          docker pull ${{ matrix.source }}
+          docker tag ${{ matrix.source }} ghcr.io/tetratelabs/getenvoy-internal:${{ matrix.target_tag }}
+          docker push ghcr.io/tetratelabs/getenvoy-internal:${{ matrix.target_tag }}

Makefile

@@ -88,7 +88,6 @@ release.dryrun:
 
 .PHONY: test
 test:
-	docker-compose up -d
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
 .PHONY: e2e

ci/e2e/darwin/install_docker.sh

@@ -38,5 +38,6 @@ while ! docker info 2> /dev/null; do
 	If Docker is not ready within 10 minutes, it's better to fail the current CI Job and let the comitter to re-start the job manually"
 done
 
-# sanity check without using docker.io as it is rate-limited
-docker run --rm quay.io/prometheus/busybox date
+# Verify install without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+docker run --rm ghcr.io/tetratelabs/getenvoy-internal:busybox date

docker-compose.yml

@@ -15,6 +15,8 @@
 version: '3'
 services:
   registry:
-    image: registry:2
+    # ghcr.io allows end-to-end (e2e) tests to avoid docker.io rate limits
+    # .github/workflows/internal-images.yml publishes this
+    image: ghcr.io/tetratelabs/getenvoy-internal:registry
     ports:
       - "5000:5000"

images/extension-builders/rust/Dockerfile

@@ -15,7 +15,9 @@
 #
 # Builder image for Envoy Wasm extensions written in Rust.
 #
-FROM rust:1.44.1
+# ghcr.io allows building without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+FROM ghcr.io/tetratelabs/getenvoy-internal:rust
 
 RUN rustup target add wasm32-unknown-unknown
 

images/extension-builders/tinygo/Dockerfile

@@ -15,8 +15,9 @@
 #
 # Builder image for Envoy Wasm extensions written in Go.
 #
-
-FROM tinygo/tinygo:0.17.0
+# ghcr.io allows building without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+FROM ghcr.io/tetratelabs/getenvoy-internal:tinygo
 
 ENV GOCACHE=/source/build/.gocache
 ENV GOMODCACHE=/source/build/.gomodcache

pkg/cmd/extension/push/cmd_test.go

@@ -15,8 +15,14 @@
 package push_test
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -29,15 +35,17 @@ import (
 // relativeWorkspaceDir points to a usable pre-initialized workspace
 const relativeWorkspaceDir = "testdata/workspace"
 
-// localRegistryWasmImageRef corresponds to a Docker container running the image "registry:2"
-// As this is not intended to be an end-to-end test, this could be improved to use a mock/fake HTTP registry instead.
-const localRegistryWasmImageRef = "localhost:5000/getenvoy/sample"
-
 // When unspecified, we default the tag to Docker's default "latest". Note: recent tools enforce qualifying this!
 const defaultTag = "latest"
 
 // TestGetEnvoyExtensionPush shows current directory is usable, provided it is a valid workspace.
 func TestGetEnvoyExtensionPush(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	_, revertWd := RequireChDir(t, relativeWorkspaceDir)
 	defer revertWd()
 
@@ -51,14 +59,19 @@ func TestGetEnvoyExtensionPush(t *testing.T) {
 
 	// Verify stdout shows the latest tag and the correct image ref
 	require.NoError(t, err, `expected no error running [%v]`, c)
-
 	require.Contains(t, stdout.String(), fmt.Sprintf(`Using default tag: %s
 Pushed %s
 digest: sha256`, defaultTag, imageRef), `unexpected stderr after running [%v]`, c)
 	require.Empty(t, stderr, `expected no stderr running [%v]`, c)
 }
 
 func TestGetEnvoyExtensionPushFailsOutsideWorkspaceDirectory(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	// Change to a non-workspace dir
 	dir, revertWd := RequireChDir(t, relativeWorkspaceDir+"/..")
 	defer revertWd()
@@ -78,6 +91,12 @@ func TestGetEnvoyExtensionPushFailsOutsideWorkspaceDirectory(t *testing.T) {
 
 // TestGetEnvoyExtensionPushWithExplicitFileOption shows we don't need to be in a workspace directory to push a wasm.
 func TestGetEnvoyExtensionPushWithExplicitFileOption(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	// Change to a non-workspace dir
 	dir, revertWd := RequireChDir(t, relativeWorkspaceDir+"/..")
 	defer revertWd()
@@ -97,3 +116,46 @@ Pushed %s:latest
 digest: sha256`, localRegistryWasmImageRef))
 	require.Empty(t, stderr, `expected no stderr running [%v]`, c)
 }
+
+// The tests above are unit tests, not end-to-end (e2e) tests. Hence, we use a mock registry instead of a real one.
+func mockRegistryServer(t *testing.T) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		statusCode := 500
+
+		body, err := ioutil.ReadAll(r.Body) // fully read the request
+		require.NoError(t, err, "Error reading body of %s %s", r.Method, r.URL.Path)
+
+		switch r.Method {
+		case "HEAD":
+			if strings.Index(r.URL.Path, "/v2/getenvoy/sample/blobs") == 0 {
+				statusCode = 404 // pretend it hasn't been uploaded, yet
+			} else if r.URL.Path == "/v2/getenvoy/sample/manifests/latest" {
+				statusCode = 404 // pretend there's no manifest either
+			}
+		case "POST":
+			if r.URL.Path == "/v2/getenvoy/sample/blobs/uploads/" {
+				statusCode = 202 // pretend we processed the data
+				w.Header().Add("Location", "/upload")
+			}
+		case "PUT":
+			if r.URL.Path == "/upload" {
+				err := r.ParseForm()
+				require.NoError(t, err, "Error parsing PUT %s", r.URL.Path)
+				require.NotEmpty(t, r.Form.Get("digest"), `Expected PUT %s to have a query parameter "digest"`, r.URL.Path)
+
+				w.Header().Add("Docker-Content-Digest", r.Form.Get("digest"))
+				statusCode = 200 // Pretend we accepted the blob
+			} else if r.URL.Path == "/v2/getenvoy/sample/manifests/latest" {
+				w.Header().Add("Docker-Content-Digest", "sha256:"+hash(body))
+				statusCode = 200 // Pretend we accepted the manifest
+			}
+		}
+		w.WriteHeader(statusCode)
+	}))
+}
+
+func hash(b []byte) string {
+	h := sha256.New()
+	h.Write(b)
+	return hex.EncodeToString(h.Sum(nil))
+}