iteration towards local dev/ci setup for arm64 compateble images

jupyter · Jul 11, 2021 · 141da2b · 141da2b
1 parent d759477
commit 141da2b
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 31 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -22,28 +22,19 @@ on:
       - ".pre-commit-config.yaml"
 
 jobs:
-  build-images:
-    name: Build Docker Images
+  build-test-publish-images:
+    name: Build, test, and publish Docker Images
     runs-on: ubuntu-latest
     permissions:
       contents: write
     if: >
       !contains(github.event.head_commit.message, 'ci skip') &&
       !contains(github.event.pull_request.title, 'ci skip')
-    steps:
-      - name: Should we push this image to a public registry?
-        run: |
-          if [ "${{ (github.ref == 'refs/heads/master') || (github.ref == 'refs/heads/main') }}" = "true" ]; then
-              # Empty => Docker Hub
-              echo "REGISTRY=" >> $GITHUB_ENV
-          else
-              echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV
-          fi
 
+    steps:
       # Setup docker to build for multiple platforms, see:
       # https://github.com/docker/build-push-action/tree/master#usage
       # https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md
-
       - name: Set up QEMU (for docker buildx)
         uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # dependabot updates to latest release
 
@@ -57,37 +48,49 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: main
+
       - name: Set Up Python
         uses: actions/setup-python@v2
         with:
           python-version: 3.x
+
       - name: Install Dev Dependencies
         run: |
           python -m pip install --upgrade pip
           make -C main dev-env
+
       - name: Build Docker Images
-        run: make -C main build-all
+        run: make -C main build-all-multi
         env:
           # Full logs for CI build
           BUILDKIT_PROGRESS: plain
+
       - name: Test Docker Images
         run: make -C main test-all
+
       - name: Clone Wiki
         uses: actions/checkout@v2
         with:
           repository: ${{github.repository}}.wiki
           path: wiki
+
       - name: Run Post-Build Hooks
-        run: make -C main hook-all
+        run: make -C main hook-all-multi
+
       - name: Push Wiki to GitHub
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
         uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release
         with:
           commit_message: "[ci skip] Automated publish for ${{github.sha}}"
           repository: wiki/
+
       - name: Login to Docker Hub
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # dependabot updates to latest release
         with:
           username: ${{secrets.DOCKERHUB_USERNAME}}
           password: ${{secrets.DOCKERHUB_TOKEN}}
+
+      - name: Push Images to DockerHub
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
+        run: make -C main push-all-multi
diff --git a/Makefile b/Makefile
@@ -4,22 +4,29 @@
 
 # Use bash for inline if-statements in arch_patch target
 SHELL:=bash
+LOCAL_REGISTRY?=localhost:5000/
+REMOTE_REGISTRY?=localhost:5000/
 OWNER?=jupyter
 
 # Need to list the images in build dependency order
 
-# These are images we can cross-build
-CROSS_IMAGES:= base-notebook \
+# Images supporting the following architectures:
+# - linux/amd64
+# - linux/arm64
+MULTI_IMAGES:= \
+	base-notebook \
 	minimal-notebook
-# These images that aren't currently supported for cross-building, your help is welcome.
-X86_IMAGES:= r-notebook \
+# Images that can only be built on the amd64 architecture (aka. x86_64)
+AMD64_ONLY_IMAGES:= \
+	r-notebook \
 	scipy-notebook \
 	tensorflow-notebook \
 	datascience-notebook \
 	pyspark-notebook \
 	all-spark-notebook
 # All of the images
-ALL_IMAGES:=base-notebook \
+ALL_IMAGES:= \
+	base-notebook \
 	minimal-notebook \
 	r-notebook \
 	scipy-notebook \
@@ -35,27 +42,53 @@ export DOCKER_BUILDKIT:=1
 help:
 	@echo "jupyter/docker-stacks"
 	@echo "====================="
-	@echo "Replace % with a stack directory name (e.g., make build-cross/minimal-notebook)"
+	@echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)"
 	@echo
 	@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-build-x86/%: DARGS?=
-build-x86/%: ## build the latest image for a stack on x86 only
-	docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64" --push
+
+
+
+
+build-amd64/%: DARGS?=
+build-amd64/%: ## build the latest image for a stack on amd64 only
+	docker build $(DARGS) --rm --force-rm -t $(notdir $@):latest,$(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER)
 	@echo -n "Built image size: "
 	@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
 
-build-cross/%: DARGS?=
-build-cross/%: ## build the latest image for a stack on x86 and ARM
-	docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push
+build-all: $(foreach I,$(ALL_IMAGES), build-amd64/$(I)) ## build all stacks
+
+# Limitations on docker buildx build (using docker/buildx 0.5.1):
+# - Can't --load and --push at the same time
+# - Can't --load multiple platforms
+#
+# What does it mean to --load?
+# - --load is a `docker buildx build` flag. It is something done by default when
+#   using the normal `docker build` command
+# - Without being loaded into the docker host, the image won't be found when you
+#   try to do `docker run` or `docker tag`..
+#
+# Workarounds due to limitations:
+# 1. We always make a dedicated amd64 build, prefixed ""
+# 2. We always also build a multi-platform image, using the normal name that
+#    won't be 
+#
+build-multi/%: DARGS?=
+build-multi/%: ## build the latest image for a stack on amd64 and arm64
+	docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64" --load
+	docker buildx build $(DARGS) --rm --force-rm -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
 	@echo -n "Built image size: "
 	@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
 
-build-all: $(foreach I,$(CROSS_IMAGES), build-cross/$(I) ) $(foreach I,$(X86_IMAGES), build-x86/$(I) ) ## build all stacks
+build-all-multi: $(foreach I,$(MULTI_IMAGES), build-multi/$(I)) $(foreach I,$(AMD64_ONLY_IMAGES), build-amd64/$(I)) ## build all stacks
+
+
 
 check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental)
 	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
-check-outdated-all: $(foreach I,$(ALL_IMAGES), check-outdated/$(I) ) ## check all the stacks for outdated conda packages
+check-outdated-all: $(foreach I,$(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated conda packages
+
+
 
 cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
 
@@ -67,6 +100,8 @@ cont-rm-all: ## remove all containers
 	@echo "Removing all containers ..."
 	-docker rm --force $(shell docker ps -a -q) 2> /dev/null
 
+
+
 dev/%: ARGS?=
 dev/%: DARGS?=-e JUPYTER_ENABLE_LAB=yes
 dev/%: PORT?=8888
@@ -76,15 +111,35 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	@pip install -r requirements-dev.txt
 
+
+
 docs: ## build HTML documentation
 	sphinx-build docs/ docs/_build/
 
+
+
+
 hook/%: WIKI_PATH?=../wiki
 hook/%: ## run post-build hooks for an image
 	python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
 	python3 -m tagging.create_manifests --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)"
+hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I)) ## run amd64 post-build hooks for all images
+
+# FIXME: `docker tag <source> <dest>` won't work without for us like this.
+#         Either we have to introduce a local registry to get a valid <source>
+#         image, or we have to list all tags when we do `docker buildx build -t
+#         <comma separated array of image:tag entries> --push` in push-multi.
+#
+#         I don't know going for a local registry is very helpful, I think it
+#         can cause complexity both by introducing it, push/pull times, and by
+#         having names prefixed with the local registry sometimes but othertimes
+#         not etc.
+#
+hook-multi/%: ## run post-build hooks for an image
+	python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)"
+hook-all-multi: $(foreach I,$(MULTI_IMAGES),hook-multi/$(I)) $(foreach I,$(AMD64_ONLY_IMAGES),hook/$(I)) ## run post-build hooks for all images
+
 
-hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
 
 img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
 
@@ -100,24 +155,37 @@ img-rm-dang: ## remove dangling images (tagged None)
 	@echo "Removing dangling images ..."
 	-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
 
+
+
 pre-commit-all: ## run pre-commit hook on all files
 	@pre-commit run --all-files || (printf "\n\n\n" && git --no-pager diff --color=always)
 
 pre-commit-install: ## set up the git hook scripts
 	@pre-commit --version
 	@pre-commit install
 
+
+
 pull/%: DARGS?=
 pull/%: ## pull a jupyter image
 	docker pull $(DARGS) $(OWNER)/$(notdir $@)
 
-pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images
+pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I)) ## pull all images
 
 push/%: DARGS?=
 push/%: ## push all tags for a jupyter image
 	docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@)
 
-push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images
+push-all: $(foreach I,$(ALL_IMAGES),push/$(I)) ## push all tagged images
+
+# FIXME: See the hook-multi step's FIXME note.
+push-multi/%: DARGS?=
+push-multi/%: ## push all tags for a jupyter image that support multiple architectures
+	docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
+
+push-all-multi: $(foreach I,$(MULTI_IMAGES),push-multi/$(I)) $(foreach I,$(AMD64_ONLY_IMAGES),push/$(I)) ## push all tagged images
+
+
 
 run/%: DARGS?=
 run/%: ## run a bash in interactive mode in a stack
@@ -127,6 +195,8 @@ run-sudo/%: DARGS?=
 run-sudo/%: ## run a bash in interactive mode as root in a stack
 	docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
+
+
 test/%: ## run tests against a stack (only common tests or common tests + specific tests)
 	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
 	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi