From 15f6e8c61e5f52606e208208c55b5426e787883b Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 15 Mar 2018 17:33:59 -0400 Subject: [PATCH] update invoker configuration to talk about two container factory options --- kubernetes/invoker/README.md | 101 ++++++++++++++---- .../invoker/{invoker.env => invoker-dcf.env} | 2 +- .../invoker/{invoker.yml => invoker-dcf.yml} | 0 kubernetes/invoker/invoker-k8scf.env | 2 +- tools/travis/build.sh | 8 +- 5 files changed, 86 insertions(+), 27 deletions(-) rename kubernetes/invoker/{invoker.env => invoker-dcf.env} (60%) rename kubernetes/invoker/{invoker.yml => invoker-dcf.yml} (100%) diff --git a/kubernetes/invoker/README.md b/kubernetes/invoker/README.md index 9e60c11a..a5e4dee3 100644 --- a/kubernetes/invoker/README.md +++ b/kubernetes/invoker/README.md @@ -1,45 +1,104 @@ Invoker ------- -# Deploying +# Overview + +The Invoker is responsible for creating and managing the containers +that OpenWhisk creates to execute the user defined functions. A key +function of the Invoker is to manage a cache of available warm +containers to minimize cold starts of user functions. +Architecturally, we support two options for deploying the Invoker +component on Kubernetes (selected by picking a +`ContainerFactoryProviderSPI` for your deployment). + 1. `DockerContainerFactory` matches the architecture used by the + non-Kubernetes deployments of OpenWhisk. In this approach, an + Invoker instance runs on every Kubernetes worker node that is + being used to execute user functions. The Invoker directly + communicates with the docker daemon running on the worker node + to create and manage the user function containers. The primary + advantages of this configuration are lower latency on container + management operations and robustness of the code paths being + used (since they are the same as in the default system). The + primary disadvantage is that it does not leverage Kubernetes to + simplify resource management, security configuration, etc. for + user containers. + 2. `KubernetesContainerFactory` is a truly Kubernetes-native design + where although the Invoker is still responsible for managing the + cache of available user containers, the Invoker relies on Kubernetes to + create, schedule, and manage the Pods that contain the user function + containers. The pros and cons of this design are roughly the + inverse of `DockerContainerFactory`. Kubernetes pod management + operations have higher latency and exercise newer code paths in + the Invoker. However, this design fully leverages Kubernetes to + manage the execution resources for user functions. -## Create config map +# Deploying -Edit invoker.env as needed to set the appropriate values for your -deployment, then create the configmap invoker.config: +## Label the worker nodes +In either approach, it is desirable to indicate which worker nodes +should be used to execute user containers. Do this by labeling each +node with `openwhisk-role=invoker`. For example, +``` +$ kubectl label nodes 127.0.0.1 openwhisk-role=invoker +``` +or for a single node cluster ``` -kubectl -n openwhisk create cm invoker.config --from-env-file=invoker.env +kubectl label nodes --all openwhisk-role=invoker ``` -## Deploy Invoker +## Deploying using the DockerContainerFactory -When deploying the Invoker, it needs to be deployed via a -[DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/). -This is because there should only ever be at most 1 Invoker -instance per Kube Node. To set these restrictions, it will be -up to the Kubernetes deployment operator to properly apply -the correct labels and taints to each required Kube node. +### Create the invoker.config config map -With the defaults in the current `invoker.yml`, you can setup a -node to run only Invoker pods with: +Edit invoker-dcf.env to make any customizations needed for your +deployment, create the config map: +``` +kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-dcf.env +``` +### Deploy the Invoker as a DaemonSet + +This will deploy an Invoker instance on every Kubernetes worker node +labeled with openwhisk-role=invoker. ``` -kubectl label nodes [node name] openwhisk-role=invoker -$ kubectl label nodes 127.0.0.1 openwhisk-role=invoker +kubectl apply -f invoker-dcf.yml ``` -Once the invoker label is applied, you can create the invokers with: +## Deploying using the KubernetesContainerFactory +The KubernetesContainerFactory can be deployed with an additional +invokerAgent that implements container suspend/resume operations on +behalf of a remote Invoker. The instructions here included deploying +the invokerAgent. If you do not want to do this, skip deploying the +invokerAgent daemonset and edit invoker-k8scf.yml to set +`CONFIG_whisk_kubernetes_invokerAgent_enabled` to `FALSE`. + +### Create the invoker.config config map + +Edit invoker-k8scf.env to make any customizations needed for your +deployment, create the config map: +``` +kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-k8scf.env +``` + +### Deploy the invokerAgent Daemonset ``` -kubectl apply -f invoker.yml +kubectl apply -f invoker-agent.yml ``` -**Important** +### Deploy the Invoker as a StatefulSet + +By default, this will deploy a single Invoker instance. Optionally +edit invoker-k8scf.yml to change the number of Invoker replicas and +then do: +``` +kubectl apply -f invoker-k8scf.yml +``` # Troubleshooting -## No invokers are deployed +## No invokers are deployed with DockerContainerFactory Verify that you actually have at least one node with the label openwhisk-role=invoker. @@ -54,5 +113,5 @@ means that the default volume hostPath values assume that the Kubernetes worker node image is Ubuntu. If containers fail to start with errors related mounting`/sys/fs/cgroup`, `/run/runc`,`/var/lib/docker/containers`, or `/var/run/docker.sock`, then you will need to change the corresponding -value in [invoker.yml](invoker.yml) to match the host operating system +value in [invoker-dcf.yml](invoker-dcf.yml) to match the host operating system running on your Kubernetes worker node. diff --git a/kubernetes/invoker/invoker.env b/kubernetes/invoker/invoker-dcf.env similarity index 60% rename from kubernetes/invoker/invoker.env rename to kubernetes/invoker/invoker-dcf.env index 75285b89..ac1346f6 100644 --- a/kubernetes/invoker/invoker.env +++ b/kubernetes/invoker/invoker-dcf.env @@ -1,4 +1,4 @@ -java_opts=-Xmx2g +java_opts=-Xmx2g -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.docker.DockerContainerFactoryProvider invoker_opts= invoker_container_network=bridge invoker_container_dns= diff --git a/kubernetes/invoker/invoker.yml b/kubernetes/invoker/invoker-dcf.yml similarity index 100% rename from kubernetes/invoker/invoker.yml rename to kubernetes/invoker/invoker-dcf.yml diff --git a/kubernetes/invoker/invoker-k8scf.env b/kubernetes/invoker/invoker-k8scf.env index 1afadbc8..2cd03bf9 100644 --- a/kubernetes/invoker/invoker-k8scf.env +++ b/kubernetes/invoker/invoker-k8scf.env @@ -1,4 +1,4 @@ -java_opts=-Xmx2g -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider -Dwhisk.spi.LogStoreProvider=whisk.core.containerpool.kubernetes.KubernetesInvokerAgentLogStoreProvider +java_opts=-Xmx2g -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider invoker_opts= invoker_container_network=bridge invoker_container_dns= diff --git a/tools/travis/build.sh b/tools/travis/build.sh index 3920d80b..8feaceb8 100755 --- a/tools/travis/build.sh +++ b/tools/travis/build.sh @@ -137,7 +137,7 @@ ROOTDIR="$SCRIPTDIR/../../" cd $ROOTDIR -# Label invoker nodes (needed for daemonset-based invoker deployment) +# Label invoker nodes (needed for DockerContainerFactory-based invoker deployment) echo "Labeling invoker node" kubectl label nodes --all openwhisk-role=invoker kubectl describe nodes @@ -210,10 +210,10 @@ pushd kubernetes/controller popd # setup the invoker -echo "Deploying invoker" +echo "Deploying invoker using DockerContainerFactory" pushd kubernetes/invoker - kubectl -n openwhisk create cm invoker.config --from-env-file=invoker.env - kubectl apply -f invoker.yml + kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-dcf.env + kubectl apply -f invoker-dcf.yml # wait until the invoker is ready deploymentHealthCheck "invoker"