From 54a44aeb1b0f87fc9fb51e5838491d20701dfb82 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 26 Jan 2018 11:48:20 -0500 Subject: [PATCH] WIP on kubernetes container pool 1. Deployment files for invoker statefulset using KubernetesContainerPool. 2. Implement simple Go invoker-agent to proxy pause/unpause operations for a remote invoker instance. --- docker/README.md | 2 + docker/invoker-agent/Dockerfile | 33 +++++ docker/invoker-agent/main.go | 90 ++++++++++++ kubernetes/invoker/invoker-agent.yml | 100 ++++++++++++++ kubernetes/invoker/invoker-k8scf.env | 9 ++ kubernetes/invoker/invoker-k8scf.yml | 197 +++++++++++++++++++++++++++ 6 files changed, 431 insertions(+) create mode 100644 docker/invoker-agent/Dockerfile create mode 100644 docker/invoker-agent/main.go create mode 100644 kubernetes/invoker/invoker-agent.yml create mode 100644 kubernetes/invoker/invoker-k8scf.env create mode 100644 kubernetes/invoker/invoker-k8scf.yml diff --git a/docker/README.md b/docker/README.md index 7be3b0ab..631214e7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,6 +11,8 @@ The built images are: * docker-pull - performs a 'docker pull' for action runtimes specified in runtimesManifest format -- used to prefetch action runtime images for invoker nodes + * invoker-agent - worker node invoker agent -- used to implement + suspend/resume operations by relying commands to local docker/containerd. * openwhisk-catalog - installs the catalog from the project incubator-openwhisk-calalog to the system namespace of the OpenWhisk deployment. diff --git a/docker/invoker-agent/Dockerfile b/docker/invoker-agent/Dockerfile new file mode 100644 index 00000000..fd7e30b4 --- /dev/null +++ b/docker/invoker-agent/Dockerfile @@ -0,0 +1,33 @@ +from ubuntu:latest + +ENV DOCKER_VERSION 1.12.0 +ENV KUBERNETES_VERSION 1.6.4 + +RUN apt-get -y update && apt-get -y install \ + wget \ + git \ + golang-go + +# Install docker client (for intractive debugging; not actually needed by agent) +RUN wget --no-verbose https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \ +tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \ +tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker-runc && \ +rm -f docker-${DOCKER_VERSION}.tgz && \ +chmod +x /usr/bin/docker && \ +chmod +x /usr/bin/docker-runc + +# Install kubernetes client (for intractive debugging; not actually needed by agent) +RUN wget --no-verbose https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && \ +chmod +x kubectl && \ +mv kubectl /usr/bin/kubectl + +RUN mkdir -p /openwhisk/src/invoker-agent + +ENV GOPATH=/openwhisk + +COPY main.go /openwhisk/src/invoker-agent + +RUN go get --insecure github.com/gorilla/mux +RUN go install invoker-agent + +CMD ["/openwhisk/bin/invoker-agent"] diff --git a/docker/invoker-agent/main.go b/docker/invoker-agent/main.go new file mode 100644 index 00000000..b499dde8 --- /dev/null +++ b/docker/invoker-agent/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "log" + "net" + "net/http" + "os" + "strings" + "time" + "github.com/gorilla/mux" +) + +var client *http.Client + +const logTime = false + +func resumeUserAction(w http.ResponseWriter, r *http.Request){ + var start time.Time + if (logTime) { + start = time.Now() + } + + vars := mux.Vars(r) + container := vars["container"] + dummy := strings.NewReader("") + resp, err := client.Post("http://localhost/containers/"+container+"/unpause", "text/plain", dummy) + if err != nil { + w.WriteHeader(500) + fmt.Fprintf(w, "Unpausing %s failed with error: %v\n", container, err) + } else if resp.StatusCode < 200 || resp.StatusCode > 299 { + w.WriteHeader(resp.StatusCode) + fmt.Fprint(w, "Unpausing %s failed with status code: %d\n", container, resp.StatusCode) + } else { + w.WriteHeader(204) // success! + } + + if (logTime) { + end := time.Now() + elapsed :=end.Sub(start) + fmt.Fprintf(os.Stdout, "Unpause took %s\n", elapsed.String()) + } +} + +func suspendUserAction(w http.ResponseWriter, r *http.Request){ + var start time.Time + if (logTime) { + start = time.Now() + } + + vars := mux.Vars(r) + container := vars["container"] + dummy := strings.NewReader("") + resp, err := client.Post("http://localhost/containers/"+container+"/pause", "text/plain", dummy) + if err != nil { + w.WriteHeader(500) + fmt.Fprintf(w, "Pausing %s failed with error: %v\n", container, err) + } else if resp.StatusCode < 200 || resp.StatusCode > 299 { + w.WriteHeader(resp.StatusCode) + fmt.Fprint(w, "Pausing %s failed with status code: %d\n", container, resp.StatusCode) + } else { + w.WriteHeader(204) // success! + } + + if (logTime) { + end := time.Now() + elapsed :=end.Sub(start) + fmt.Fprintf(os.Stdout, "Pause took %s\n", elapsed.String()) + } +} + +func handleRequests() { + myRouter := mux.NewRouter().StrictSlash(true) + myRouter.HandleFunc("/suspend/{container}", suspendUserAction) + myRouter.HandleFunc("/resume/{container}", resumeUserAction) + log.Fatal(http.ListenAndServe(":3233", myRouter)) +} + +func main() { + // Open http client to /var/run/docker.sock + fd := func (proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", "/var/run/docker.sock") + } + tr := &http.Transport{ + Dial: fd, + } + client = &http.Client{Transport: tr} + + handleRequests() +} diff --git a/kubernetes/invoker/invoker-agent.yml b/kubernetes/invoker/invoker-agent.yml new file mode 100644 index 00000000..d4884d0b --- /dev/null +++ b/kubernetes/invoker/invoker-agent.yml @@ -0,0 +1,100 @@ +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: invoker-agent + namespace: openwhisk + labels: + name: invoker-agent +spec: + template: + metadata: + labels: + name: invoker-agent + spec: + restartPolicy: Always + hostNetwork: true + + # run only on nodes labeled with openwhisk-role=invoker + # TODO: disabled affinity until user-action pods are + # created with the same affinity rules. + # Requires extension to upstream kube java client + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: openwhisk-role + # operator: In + # values: + # - invoker + + volumes: + - name: cgroup + hostPath: + path: "/sys/fs/cgroup" + - name: runc + hostPath: + path: "/run/runc" + - name: dockerrootdir + hostPath: + path: "/var/lib/docker/containers" + - name: dockersock + hostPath: + path: "/var/run/docker.sock" + + initContainers: + - name: docker-pull-runtimes + imagePullPolicy: Always + image: openwhisk/kube-docker-pull + volumeMounts: + - name: dockersock + mountPath: "/var/run/docker.sock" + env: + # action runtimes + - name: "RUNTIMES_MANIFEST" + valueFrom: + configMapKeyRef: + name: whisk.runtimes + key: runtimes + + containers: + - name: invoker-agent + imagePullPolicy: Always + image: dgrove/invoker-agent + securityContext: + privileged: true + ports: + # IANA port 3233 "whisker" for "WhiskerControl" ;) + - name: agent + containerPort: 3233 + hostPort: 3233 + volumeMounts: + - name: cgroup + mountPath: "/sys/fs/cgroup" + - name: runc + mountPath: "/run/runc" + - name: dockersock + mountPath: "/var/run/docker.sock" + - name: dockerrootdir + mountPath: "/containers" + env: + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: invoker-agent-netpol + namespace: openwhisk +spec: + podSelector: + matchLabels: + name: invoker-agent + ingress: + # Allow invoker to connect to invoker-agent + - from: + - podSelector: + matchLabels: + name: invoker + ports: + - port: 3233 diff --git a/kubernetes/invoker/invoker-k8scf.env b/kubernetes/invoker/invoker-k8scf.env new file mode 100644 index 00000000..2cd03bf9 --- /dev/null +++ b/kubernetes/invoker/invoker-k8scf.env @@ -0,0 +1,9 @@ +java_opts=-Xmx2g -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider +invoker_opts= +invoker_container_network=bridge +invoker_container_dns= +invoker_use_runc=false +docker_image_prefix=openwhisk +docker_image_tag=latest +docker_registry= +invoker_logs_dir= diff --git a/kubernetes/invoker/invoker-k8scf.yml b/kubernetes/invoker/invoker-k8scf.yml new file mode 100644 index 00000000..4ba97a84 --- /dev/null +++ b/kubernetes/invoker/invoker-k8scf.yml @@ -0,0 +1,197 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: invoker + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + namespace: openwhisk + name: invoker +rules: +- apiGroups: ["extensions"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: invoker-rbac + namespace: openwhisk +subjects: +- kind: ServiceAccount + name: invoker + namespace: openwhisk +roleRef: + kind: Role + name: invoker + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: invoker + namespace: openwhisk + labels: + name: invoker +spec: + replicas: 1 + serviceName: invoker + template: + metadata: + labels: + name: invoker + spec: + serviceAccountName: invoker + restartPolicy: Always + + affinity: + # prefer to run on an invoker node (only prefer because of single node clusters) + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: openwhisk-role + operator: In + values: + - invoker + # do not allow more than 1 invoker instance to run on a node + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - invoker + topologyKey: "kubernetes.io/hostname" + + containers: + - name: invoker + imagePullPolicy: Always + image: dgrove/invoker + command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /init.sh" ] + ports: + - name: invoker + containerPort: 8080 + env: + - name: "PORT" + value: "8080" + + # Invoker name is name of pod (invoker-0, invoker-1, etc). + - name: "INVOKER_NAME" + valueFrom: + fieldRef: + fieldPath: metadata.name + + - name: "WHISK_API_HOST_NAME" + valueFrom: + configMapKeyRef: + name: whisk.ingress + key: api_host + + # Docker-related options + - name: "INVOKER_USE_RUNC" + value: "FALSE" + - name: "DOCKER_IMAGE_PREFIX" + valueFrom: + configMapKeyRef: + name: invoker.config + key: docker_image_prefix + - name: "DOCKER_IMAGE_TAG" + valueFrom: + configMapKeyRef: + name: invoker.config + key: docker_image_tag + - name: "DOCKER_REGISTRY" + valueFrom: + configMapKeyRef: + name: invoker.config + key: docker_registry + + # action runtimes + - name: "RUNTIMES_MANIFEST" + valueFrom: + configMapKeyRef: + name: whisk.runtimes + key: runtimes + + # extra JVM arguments + - name: "JAVA_OPTS" + valueFrom: + configMapKeyRef: + name: invoker.config + key: java_opts + + # extra Invoker arguments + - name: "INVOKER_OPTS" + valueFrom: + configMapKeyRef: + name: invoker.config + key: invoker_opts + + # Recommend using "" because logs should go to stdout on kube + - name: "WHISK_LOGS_DIR" + valueFrom: + configMapKeyRef: + name: invoker.config + key: invoker_logs_dir + + # properties for Kafka connection + - name: "KAFKA_HOSTS" + value: "$(KAFKA_SERVICE_HOST):$(KAFKA_SERVICE_PORT_KAFKA)" + + # properties for zookeeper connection + - name: "ZOOKEEPER_HOSTS" + value: "$(ZOOKEEPER_SERVICE_HOST):$(ZOOKEEPER_SERVICE_PORT_ZOOKEEPER)" + + # properties for DB connection + - name: "DB_USERNAME" + valueFrom: + secretKeyRef: + name: db.auth + key: db_username + - name: "DB_PASSWORD" + valueFrom: + secretKeyRef: + name: db.auth + key: db_password + - name: "DB_PROTOCOL" + valueFrom: + configMapKeyRef: + name: db.config + key: db_protocol + - name: "DB_HOST" + value: "$(COUCHDB_SERVICE_HOST)" + - name: "DB_PORT" + value: "$(COUCHDB_SERVICE_PORT_COUCHDB)" + - name: "DB_PROVIDER" + valueFrom: + configMapKeyRef: + name: db.config + key: db_provider + - name: "DB_WHISK_ACTIVATIONS" + valueFrom: + configMapKeyRef: + name: db.config + key: db_whisk_activations + - name: "DB_WHISK_ACTIONS" + valueFrom: + configMapKeyRef: + name: db.config + key: db_whisk_actions + - name: "DB_WHISK_AUTHS" + valueFrom: + configMapKeyRef: + name: db.config + key: db_whisk_auths