diff --git a/build/Makefile b/build/Makefile index f623bb42cf..afc3c5f594 100644 --- a/build/Makefile +++ b/build/Makefile @@ -63,6 +63,26 @@ sidecar_tag = $(REGISTRY)/gameservers-sidecar:$(VERSION) go_version_flags = -ldflags "-X github.com/agonio/agon/pkg.Version=$(VERSION)" +# TODO: update documentation, variables, WSL admin, targets + +# ___ ____ ___ _ _ +# / _ \/ ___| |_ _|_ __ ___| |_ _ __| | ___ +# | | | \___ \ | || '_ \ / __| | | | |/ _` |/ _ \ +# | |_| |___) | | || | | | (__| | |_| | (_| | __/ +# \___/|____/ |___|_| |_|\___|_|\__,_|\__,_|\___| +# + +uname := $(shell uname -s) +ifneq ($(findstring Microsoft,$(shell uname -r)),) + osinclude := windows.mk +else ifeq ($(uname),Linux) + osinclude := linux.mk +else ifeq ($(uname),Darwin) + osinclude := osx.mk +endif + +include ./includes/$(osinclude) + # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ # | |/ _` | '__/ _` |/ _ \ __/ __| @@ -168,7 +188,7 @@ clean-build-image: docker rmi $(build_tag) ensure-build-config: - -mkdir -p $(build_path)/.kube + -mkdir -p $(KUBEPATH) -mkdir -p $(build_path)/.config/gcloud # create the build image if it doesn't exist @@ -237,56 +257,36 @@ clean-gcloud-config: # |_| |_|_|_| |_|_|_|\_\\__,_|_.__/ \___| # -# Switches to an agon profile, and starts a kubernetes cluster -# of the right version. Also mounts the project directory into minikube, -# so that the build tools will work. +# Switches to an "agon" profile, and starts a kubernetes cluster +# of the right version. # -# Use DRIVER variable to change the VM driver (default virtualbox) if you so desire. -minikube-test-cluster: DRIVER := virtualbox +# Use MINIKUBE_DRIVER variable to change the VM driver +# (defaults virtualbox for Linux and OSX, hyperv for windows) if you so desire. minikube-test-cluster: minikube-agon-profile - minikube start --kubernetes-version v1.8.0 --vm-driver $(DRIVER) - $(MAKE) minikube-ensure-build-image - minikube mount $(agon_path):$(agon_path) + $(MINIKUBE) start --kubernetes-version v1.8.0 --vm-driver $(MINIKUBE_DRIVER) + $(MAKE) minikube-post-start # switch to the agon cluster minikube-agon-profile: - minikube profile $(MINIKUBE_PROFILE) + $(MINIKUBE) profile $(MINIKUBE_PROFILE) # Connecting to minikube requires so enhanced permissions, so use this target # instead of `make shell` to start an interactive shell for development on minikube. minikube-shell: ensure-build-image - eval $$(minikube docker-env --unset) && \ - $(MAKE) shell ARGS="--network=host -v ~/.minikube:$(HOME)/.minikube" - -# Convenience target to build Agon's docker images directly on minikube. -minikube-build: minikube-ensure-build-image - eval $$(minikube docker-env) && \ - $(MAKE) build-images - -# ensure minikube has the build image, if not, grab it -minikube-ensure-build-image: ensure-build-image - @if [ -z $$(minikube ssh -- docker images -q $(build_tag)) ]; then\ - echo "Could not find $(build_tag) image. Transferring...";\ - $(MAKE) minikube-transfer TAG=$(build_tag);\ - fi + $(MAKE) shell ARGS="--network=host -v $(minikube_cert_mount)" -# Instead of building Agon's docker images inside minikube, -# use this command to push the local images that have already been built -# via `make build` or `make build-images`. -# -# Depending on the virtualisation driver/configuration, -# it may be faster to build locally and push, rather than building directly on minikube. +# Push the local Agon Docker images that have already been built +# via `make build` or `make build-images` into the "agon" minikube instance. minikube-push: - $(MAKE) minikube-transfer TAG=$(sidecar_tag) - $(MAKE) minikube-transfer TAG=$(controller_tag) + $(MAKE) minikube-transfer-image TAG=$(sidecar_tag) + $(MAKE) minikube-transfer-image TAG=$(controller_tag) # Installs the current development version of Agon into the Kubernetes cluster. # Use this instead of `make install`, as it disables PullAlways on the install.yaml minikube-install: ensure-build-image - eval $$(minikube docker-env --unset) && \ - $(MAKE) install ARGS="--network=host -v ~/.minikube:/$(HOME)/.minikube" ALWAYS_PULL_SIDECAR=false IMAGE_PULL_POLICY=IfNotPresent + $(MAKE) install ARGS="--network=host -v $(minikube_cert_mount)" ALWAYS_PULL_SIDECAR=false IMAGE_PULL_POLICY=IfNotPresent -# convenience target for transferring images into minikube -minikube-transfer: - eval $$(minikube docker-env --unset) && \ - docker save $(TAG) | (eval $$(minikube docker-env) && docker load) \ No newline at end of file +# Convenience target for transferring images into minikube. +# Use TAG to specify the image to transfer into minikube +minikube-transfer-image: + docker save $(TAG) | ($(MINIKUBE_DOCKER_ENV) && docker load) \ No newline at end of file diff --git a/build/README.md b/build/README.md index 9490e24064..5a2d21b51b 100644 --- a/build/README.md +++ b/build/README.md @@ -37,6 +37,8 @@ tasks you may wish to accomplish. ### Linux - Install Make, either via `apt install make` or `yum install make` depending on platform. - [Install Docker](https://docs.docker.com/engine/installation/) for your Linux platform. +- (optional) Minikube will require [VirtualBox](https://www.virtualbox.org) and will need to be installed if you wish + to develop on Minikube ### Windows Building and developing Agon requires you to use the @@ -54,21 +56,20 @@ as this makes it easy to create a (relatively) cross platform development and bu - Agon will need to be cloned somewhere on your `/c` (or drive of your choice) path, as that is what Docker will support mounts from - All interaction with Agon must be on the `/c` (or drive of your choice) path, otherwise Docker mounts will not work - Now the `make` commands can all be run from within your WSL shell - -Building and testing Agon will now work, as will developing on GKE. - -Issues with building and developing on Minikube are currently to be expected, but Agon will run on Minikube. - -You can see progress on this on the [Build Agon on Windows](https://github.com/googleprivate/agon/issues/47) ticket. +- (optional) Minikube is supported via the [HyperV](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/index) + driver - the same virtualisation platform as the Docker installation. +- **Note**: If you want to dev and test with Minikube, you **must** run WSL as Administrator, otherwise Minikube can't control HyperV. ### OSX - Install Make, `brew install make`, if it's not installed already - Install [Docker for Mac](https://docs.docker.com/docker-for-mac/install/) +- (optional) Minikube will require [VirtualBox](https://www.virtualbox.org) and will need to be installed if you wish + to develop on Minikube This has currently yet to be tested, but should have few issues around testing, building and running on GKE. -Issues with building and developing on Minikube are currently to be expected, but Agon will run on Minikube. +Issues with building and developing on Minikube are currently expected, due to lack of testing, but Agon will run on Minikube. Testing on OSX and reporting bugs are appreciated. @@ -169,10 +170,11 @@ need to be replaced by Minikube specific targets. First, [install Minikube](https://github.com/kubernetes/minikube#installation), which may also require you to install a virtualisation solution, such as [VirtualBox](https://www.virtualbox.org) as well. +Check the [Building on Different Platforms](#building-on-different-platforms) above for details on what virtualisation +solution to use. -Next we will create the Agon Minikube cluster. Run `make minikube-test-cluster` to create an `agon` profile, -create a Kubernetes cluster under this profile of the supported version, -and mount the development code inside the Minikube instance so we are able to build Agon inside Minikube. +Next we will create the Agon Minikube cluster. Run `make minikube-test-cluster` to create the `agon` profile, +and a Kubernetes cluster of the supported version under this profile. This will also install the kubectl authentication credentials in `~/.kube`, and set the [`kubectl` context](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) @@ -182,36 +184,31 @@ Great! Now we are setup, let's try out the development shell, and see if our `ku Run `make minikube-shell` to enter the development shell. You should see a bash shell that has you as the root user. Enter `kubectl get pods` and press enter. You should see that you have no resources currently, but otherwise see no errors. -Assuming that all works, let's exit the shell by typing `exit` and hitting enter, and look at a couple of -options for building, pushing and installing Agon next. - -There are two options for building Agon, and depending on your virtualisation solution and its configuration -each has it's pros and cons +Assuming that all works, let's exit the shell by typing `exit` and hitting enter, and look at a building, pushing and +installing Agon on Minikube next. -#### Building directly on Minikube -Since Minikube allows you to [reuse its Docker daemon](https://github.com/kubernetes/minikube/blob/master/docs/reusing_the_docker_daemon.md) -we can build our images to run Agon directly on Minikube! - -To do this, run `make minikube-build`, which will transfer the build image into the cluster -and run the `build-images` target on the Minikube instance, creating the images required to run Agon. - -Again depending on your virtualisation layer, you may want to configure it to allow it to have access to more -cores and/or memory than the default, to allow for faster compilation (or for it to compile at all). - -#### Pushing locally built images to Minikube You may remember in the first part of this walkthrough, we ran `make build`, which created all the images and binaries -we needed to work with Agon locally. So instead of rebuilding them, can we push them straight into Minikube? - -You bet we can! +we needed to work with Agon locally. We can push these images them straight into Minikube very easily! Run `make minikube-push` which will send all of Agon's docker images from your local Docker into the Agon Minikube instance. -This may be better option if you find building on Minikube slow, or you just prefer to build locally. - Now that the images are pushed, to install the development version, -run `make minikube-install` and Agon will install the images that you built and pushed to the Agon Minikube instance -created at the beginning of this section. (if you want to see the resulting installation yaml, you can find it in `build/.install.yaml`) +run `make minikube-install` and Agon will install the images that you built and pushed to the Agon Minikube instance +(if you want to see the resulting installation yaml, you can find it in `build/.install.yaml`). + +It's worth noting that Minikube does let you [reuse its Docker daemon](https://github.com/kubernetes/minikube/blob/master/docs/reusing_the_docker_daemon.md), +and build directly on Minikube, but in this case this approach is far simpler, +and makes cross-platform support for the build system much easier. + +If you find you also want to push your own images into Minikube, +the convenience make target `make minikube-transfer-image` can be run with the `TAG` argument specifying +the tag of the Docker image you wish to transfer into Minikube. + +For example: +```bash +$ make minikube-transfer-image TAG=myimage:0.1 +``` ### Next Steps @@ -320,18 +317,14 @@ Since Minikube runs locally, there are some targets that need to be used instead #### `minikube-test-cluster` Switches to an "agon" profile, and starts a kubernetes cluster -of the right version. Also mounts the project directory into Minikube, -so that the build tools will work. - -Use DRIVER variable to change the VM driver (default virtualbox) if you so desire. +of the right version. -#### `minikube-build` -Convenience target to build Agon's docker images directly on Minikube. +Use MINIKUBE_DRIVER variable to change the VM driver +(defaults virtualbox for Linux and OSX, hyperv for windows) if you so desire. #### `minikube-push` -Instead of building Agon's docker images inside Minikube, -use this command to push the local images that have already been built -via `make build` or `make build-images`. +Push the local Agon Docker images that have already been built +via `make build` or `make build-images` into the "agon" minikube instance. #### `minikube-install` Installs the current development version of Agon into the Kubernetes cluster. @@ -341,6 +334,6 @@ Use this instead of `make install`, as it disables PullAlways on the install.yam Connecting to Minikube requires so enhanced permissions, so use this target instead of `make shell` to start an interactive shell for development on Minikube. -Depending on the virtualisation driver/configuration, -it may be faster to build locally and push, rather than building directly on Minikube. - +#### `minikube-transfer-image` +Convenience target for transferring images into minikube. +Use TAG to specify the image to transfer into minikube diff --git a/build/includes/linux.mk b/build/includes/linux.mk new file mode 100644 index 0000000000..0b9fd1b7f2 --- /dev/null +++ b/build/includes/linux.mk @@ -0,0 +1,44 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Include for Linux operating System +# + +# __ __ _ _ _ +# \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ +# \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| +# \ V / (_| | | | | (_| | |_) | | __\__ \ +# \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ +# + +# Minikube executable +MINIKUBE ?= minikube +# Default minikube driver +MINIKUBE_DRIVER ?= virtualbox +# set docker env for minikube +MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env) + +# minikube shell mount for certificates +minikube_cert_mount := ~/.minikube:$(HOME)/.minikube + +# _____ _ +# |_ _|_ _ _ __ __ _ ___| |_ ___ +# | |/ _` | '__/ _` |/ _ \ __/ __| +# | | (_| | | | (_| | __/ |_\__ \ +# |_|\__,_|_| \__, |\___|\__|___/ +# |___/ + +# Does not do anything +minikube-post-start: \ No newline at end of file diff --git a/build/includes/osx.mk b/build/includes/osx.mk new file mode 100644 index 0000000000..7d30fd8cb7 --- /dev/null +++ b/build/includes/osx.mk @@ -0,0 +1,44 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Include for OSX operating System +# + +# __ __ _ _ _ +# \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ +# \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| +# \ V / (_| | | | | (_| | |_) | | __\__ \ +# \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ +# + +# Minikube executable +MINIKUBE ?= minikube +# Default minikube driver +MINIKUBE_DRIVER ?= virtualbox +# set docker env for minikube +MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env) + +# minikube shell mount for certificates +minikube_cert_mount := ~/.minikube:$(HOME)/.minikube + +# _____ _ +# |_ _|_ _ _ __ __ _ ___| |_ ___ +# | |/ _` | '__/ _` |/ _ \ __/ __| +# | | (_| | | | (_| | __/ |_\__ \ +# |_|\__,_|_| \__, |\___|\__|___/ +# |___/ + +# Does nothing +minikube-post-start: \ No newline at end of file diff --git a/build/includes/windows.mk b/build/includes/windows.mk new file mode 100644 index 0000000000..12dd571a52 --- /dev/null +++ b/build/includes/windows.mk @@ -0,0 +1,60 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Include for Windows, operating under WSL +# + +# __ __ _ _ _ +# \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ +# \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| +# \ V / (_| | | | | (_| | |_) | | __\__ \ +# \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ +# + +# Minikube executable +MINIKUBE ?= minikube.exe +# Default minikube driver +MINIKUBE_DRIVER ?= hyperv +# set docker env for minikube +MINIKUBE_DOCKER_ENV ?= eval $$($(MINIKUBE) docker-env --shell=bash) && \ + export DOCKER_CERT_PATH=$$(echo $$DOCKER_CERT_PATH | $(win_to_wsl_path)) + +# minikube shell mount for certificates +minikube_cert_mount = $(cert_path):$(cert_path) + +# transform the path from windows to WSL +win_to_wsl_path := sed -e 's|\([A-Z]\):|/\L\1|' -e 's|\\|/|g' + +# find the cert path +cert_path = $(realpath $(shell $(MINIKUBE) docker-env --shell bash | grep DOCKER_CERT_PATH | awk -F "=" '{ print $$2 }' | sed 's/"//g' | $(win_to_wsl_path))/..) + +# _____ _ +# |_ _|_ _ _ __ __ _ ___| |_ ___ +# | |/ _` | '__/ _` |/ _ \ __/ __| +# | | (_| | | | (_| | __/ |_\__ \ +# |_|\__,_|_| \__, |\___|\__|___/ +# |___/ + +# Sets minikube credentials +minikube-post-start: + echo "Creating minikube credentials" + export CERT_PATH=$(cert_path) && \ + docker run --rm $(common_mounts) $(ARGS) $(build_tag) kubectl config set-cluster $(MINIKUBE_PROFILE) \ + --certificate-authority=$$CERT_PATH/ca.crt --server=https://$$($(MINIKUBE) ip):8443 && \ + docker run --rm $(common_mounts) $(ARGS) $(build_tag) kubectl config set-credentials $(MINIKUBE_PROFILE) \ + --client-certificate=$$CERT_PATH/client.crt --client-key=$$CERT_PATH/client.key + docker run --rm $(common_mounts) $(ARGS) $(build_tag) kubectl config set-context $(MINIKUBE_PROFILE) \ + --cluster=$(MINIKUBE_PROFILE) --user=$(MINIKUBE_PROFILE) + docker run --rm $(common_mounts) $(ARGS) $(build_tag) kubectl config use-context $(MINIKUBE_PROFILE)