WIP on kubernetes container pool

1. Deployment files for invoker statefulset using KubernetesContainerPool.
2. Implement simple Go invoker-agent to proxy pause/unpause operations
   for a remote invoker instance.

docker/README.md

@@ -11,6 +11,8 @@ The built images are:
   * docker-pull - performs a 'docker pull' for action runtimes
     specified in runtimesManifest format -- used to prefetch
     action runtime images for invoker nodes
+  * invoker-agent - worker node invoker agent -- used to implement
+    suspend/resume operations by relying commands to local docker/containerd.
   * openwhisk-catalog - installs the catalog from the project
     incubator-openwhisk-calalog to the system namespace of the
     OpenWhisk deployment.

docker/invoker-agent/Dockerfile

@@ -0,0 +1,33 @@
+from ubuntu:latest
+
+ENV DOCKER_VERSION 1.12.0
+ENV KUBERNETES_VERSION 1.6.4
+
+RUN apt-get -y update && apt-get -y install \
+  wget \
+  git \
+  golang-go
+
+# Install docker client (for intractive debugging; not actually needed by agent)
+RUN wget --no-verbose https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \
+tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \
+tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker-runc && \
+rm -f docker-${DOCKER_VERSION}.tgz && \
+chmod +x /usr/bin/docker && \
+chmod +x /usr/bin/docker-runc
+
+# Install kubernetes client (for intractive debugging; not actually needed by agent)
+RUN wget --no-verbose https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && \
+chmod +x kubectl && \
+mv kubectl /usr/bin/kubectl
+
+RUN mkdir -p /openwhisk/src/invoker-agent
+
+ENV GOPATH=/openwhisk
+
+COPY main.go /openwhisk/src/invoker-agent
+
+RUN go get --insecure github.com/gorilla/mux
+RUN go install invoker-agent
+
+CMD ["/openwhisk/bin/invoker-agent"]

docker/invoker-agent/main.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+    "fmt"
+    "log"
+    "net"
+    "net/http"
+    "os"
+    "strings"
+    "time"
+    "github.com/gorilla/mux"
+)
+
+var client *http.Client
+
+const logTime = false
+
+func resumeUserAction(w http.ResponseWriter, r *http.Request){
+    var start time.Time
+    if (logTime) {
+        start = time.Now()
+    }
+
+    vars := mux.Vars(r)
+    container := vars["container"]
+    dummy := strings.NewReader("")
+    resp, err := client.Post("http://localhost/containers/"+container+"/unpause", "text/plain", dummy)
+    if err != nil {
+      w.WriteHeader(500)
+      fmt.Fprintf(w, "Unpausing %s failed with error: %v\n", container, err)
+    } else if resp.StatusCode < 200 || resp.StatusCode > 299 {
+      w.WriteHeader(resp.StatusCode)
+      fmt.Fprint(w, "Unpausing %s failed with status code: %d\n", container, resp.StatusCode)
+    } else {
+      w.WriteHeader(204) // success!
+    }
+
+    if (logTime) {
+        end := time.Now()
+        elapsed :=end.Sub(start)
+        fmt.Fprintf(os.Stdout, "Unpause took %s\n", elapsed.String())
+    }
+}
+
+func suspendUserAction(w http.ResponseWriter, r *http.Request){
+    var start time.Time
+    if (logTime) {
+        start = time.Now()
+    }
+
+    vars := mux.Vars(r)
+    container := vars["container"]
+    dummy := strings.NewReader("")
+    resp, err := client.Post("http://localhost/containers/"+container+"/pause", "text/plain", dummy)
+    if err != nil {
+      w.WriteHeader(500)
+      fmt.Fprintf(w, "Pausing %s failed with error: %v\n", container, err)
+    } else if resp.StatusCode < 200 || resp.StatusCode > 299 {
+      w.WriteHeader(resp.StatusCode)
+      fmt.Fprint(w, "Pausing %s failed with status code: %d\n", container, resp.StatusCode)
+    } else {
+      w.WriteHeader(204) // success!
+    }
+
+    if (logTime) {
+        end := time.Now()
+        elapsed :=end.Sub(start)
+        fmt.Fprintf(os.Stdout, "Pause took %s\n", elapsed.String())
+    }
+}
+
+func handleRequests() {
+    myRouter := mux.NewRouter().StrictSlash(true)
+    myRouter.HandleFunc("/suspend/{container}", suspendUserAction)
+    myRouter.HandleFunc("/resume/{container}", resumeUserAction)
+    log.Fatal(http.ListenAndServe(":3233", myRouter))
+}
+
+func main() {
+    // Open http client to /var/run/docker.sock
+    fd := func (proto, addr string) (conn net.Conn, err error) {
+        return net.Dial("unix", "/var/run/docker.sock")
+    }
+    tr := &http.Transport{
+        Dial: fd,
+    }
+    client = &http.Client{Transport: tr}
+
+    handleRequests()
+}

kubernetes/invoker/invoker-agent.yml

@@ -0,0 +1,100 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: invoker-agent
+  namespace: openwhisk
+  labels:
+    name: invoker-agent
+spec:
+  template:
+    metadata:
+      labels:
+        name: invoker-agent
+    spec:
+      restartPolicy: Always
+      hostNetwork: true
+
+      # run only on nodes labeled with openwhisk-role=invoker
+      # TODO: disabled affinity until user-action pods are
+      #       created with the same affinity rules.
+      #       Requires extension to upstream kube java client
+      # affinity:
+      #   nodeAffinity:
+      #     requiredDuringSchedulingIgnoredDuringExecution:
+      #       nodeSelectorTerms:
+      #       - matchExpressions:
+      #         - key: openwhisk-role
+      #           operator: In
+      #           values:
+      #           - invoker
+
+      volumes:
+      - name: cgroup
+        hostPath:
+          path: "/sys/fs/cgroup"
+      - name: runc
+        hostPath:
+          path: "/run/runc"
+      - name: dockerrootdir
+        hostPath:
+          path: "/var/lib/docker/containers"
+      - name: dockersock
+        hostPath:
+          path: "/var/run/docker.sock"
+
+      initContainers:
+      - name: docker-pull-runtimes
+        imagePullPolicy: Always
+        image: openwhisk/kube-docker-pull
+        volumeMounts:
+        - name: dockersock
+          mountPath: "/var/run/docker.sock"
+        env:
+          # action runtimes
+          - name: "RUNTIMES_MANIFEST"
+            valueFrom:
+              configMapKeyRef:
+                name: whisk.runtimes
+                key: runtimes
+
+      containers:
+      - name: invoker-agent
+        imagePullPolicy: Always
+        image: dgrove/invoker-agent
+        securityContext:
+          privileged: true
+        ports:
+         # IANA port 3233 "whisker" for "WhiskerControl"  ;)
+        - name: agent
+          containerPort: 3233
+          hostPort: 3233
+        volumeMounts:
+        - name: cgroup
+          mountPath: "/sys/fs/cgroup"
+        - name: runc
+          mountPath: "/run/runc"
+        - name: dockersock
+          mountPath: "/var/run/docker.sock"
+        - name: dockerrootdir
+          mountPath: "/containers"
+        env:
+
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: invoker-agent-netpol
+  namespace: openwhisk
+spec:
+  podSelector:
+    matchLabels:
+      name: invoker-agent
+  ingress:
+  # Allow invoker to connect to invoker-agent
+  - from:
+    - podSelector:
+        matchLabels:
+          name: invoker
+    ports:
+     - port: 3233

kubernetes/invoker/invoker-k8scf.env

@@ -0,0 +1,9 @@
+java_opts=-Xmx2g -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider
+invoker_opts=
+invoker_container_network=bridge
+invoker_container_dns=
+invoker_use_runc=false
+docker_image_prefix=openwhisk
+docker_image_tag=latest
+docker_registry=
+invoker_logs_dir=