Dashboard V1

.gitignore

@@ -2,7 +2,8 @@
 # only so we exclude them.
 bin/
 vendor/
-
+node_modules/
+build/
 .vscode/
 
 # Compiled python files.

README.md

@@ -95,6 +95,25 @@ TfJob requires Kubernetes >= 1.8
     PASSED: tf-job-tfjob-test-pqxkwk
     ```
 
+### Installing `tensorflow/k8s`'s Dashboard
+
+> **Caution: the dashboard is in very early development stage!**
+
+`tensorflow/k8s` also includes a dashboard allowing you to monitor and create `TfJobs` through a web UI.  
+To deploy the dashboard, set `dashboard.install` to `true`.  
+Note that by default the dashboard will only be accessible from within the cluster or by proxying, as the default `ServiceType` is `ClusterIP`. 
+If you wish to expose the dashboard through an external IP, set `dashboard.serviceType` to `LoadBalancer`.
+
+So, for example, if you want to enable the dashboard, and also want to expose it externally, you would do:
+
+```
+CHART=https://storage.googleapis.com/tf-on-k8s-dogfood-releases/latest/tf-job-operator-chart-latest.tgz
+helm install ${CHART} -n tf-job --wait --replace --set cloud=<gke or azure>,dashboard.install=true,dashboard.serviceType=LoadBalancer
+```
+
+This sould create a service named `tf-job-dashboard` as well as an additional deployment named `tf-job-dashboard`.
+
+
 ### Configuring the CRD
 
 The CRD must be configured properly to work with your specific Kubernetes cluster.

dashboard/backend/client/manager.go

@@ -0,0 +1,45 @@
+package client
+
+import (
+	"github.com/tensorflow/k8s/pkg/util/k8sutil"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+const (
+	CRDGroup   = "mlkube.io"
+	CRDVersion = "v1beta1"
+)
+
+type ClientManager struct {
+	restCfg     *rest.Config
+	ClientSet   *kubernetes.Clientset
+	TfJobClient *k8sutil.TfJobRestClient
+}
+
+func (c *ClientManager) init() {
+	restCfg, err := k8sutil.GetClusterConfig()
+	if err != nil {
+		panic(err)
+	}
+	c.restCfg = restCfg
+
+	clientset, err := kubernetes.NewForConfig(c.restCfg)
+	if err != nil {
+		panic(err.Error())
+	}
+	c.ClientSet = clientset
+
+	tfJobClient, err := k8sutil.NewTfJobClient()
+	if err != nil {
+		panic(err)
+	}
+	c.TfJobClient = tfJobClient
+}
+
+func NewClientManager() (ClientManager, error) {
+	cm := ClientManager{}
+	cm.init()
+
+	return cm, nil
+}

dashboard/backend/handler/apihandler.go

@@ -0,0 +1,170 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+
+	restful "github.com/emicklei/go-restful"
+	"github.com/tensorflow/k8s/dashboard/backend/client"
+	"github.com/tensorflow/k8s/pkg/spec"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type APIHandler struct {
+	cManager client.ClientManager
+}
+
+//add tensorboard ips, add azure file / gs links
+type TfJobDetail struct {
+	TfJob     *spec.TfJob `json:"tfJob"`
+	TbService *v1.Service `json:"tbService"`
+	Pods      []v1.Pod    `json:"pods"`
+}
+
+type TfJobList struct {
+	tfJobs []spec.TfJob `json:"TfJobs"`
+}
+
+func CreateHTTPAPIHandler(client client.ClientManager) (http.Handler, error) {
+	apiHandler := APIHandler{
+		cManager: client,
+	}
+
+	wsContainer := restful.NewContainer()
+	wsContainer.EnableContentEncoding(true)
+
+	cors := restful.CrossOriginResourceSharing{
+		ExposeHeaders:  []string{"X-My-Header"},
+		AllowedHeaders: []string{"Content-Type", "Accept"},
+		AllowedMethods: []string{"GET", "POST", "DELETE"},
+		CookiesAllowed: false,
+		Container:      wsContainer,
+	}
+	wsContainer.Filter(cors.Filter)
+	wsContainer.Filter(wsContainer.OPTIONSFilter)
+
+	apiV1Ws := new(restful.WebService)
+
+	apiV1Ws.Path("/api").
+		Consumes(restful.MIME_JSON).
+		Produces(restful.MIME_JSON)
+
+	apiV1Ws.Route(
+		apiV1Ws.GET("/tfjob").
+			To(apiHandler.handleGetTfJobs).
+			Writes(TfJobList{}))
+
+	apiV1Ws.Route(
+		apiV1Ws.GET("/tfjob/{namespace}/{tfjob}").
+			To(apiHandler.handleGetTfJobDetail).
+			Writes(TfJobDetail{}))
+
+	apiV1Ws.Route(
+		apiV1Ws.POST("/tfjob").
+			To(apiHandler.handleDeploy).
+			Reads(spec.TfJob{}).
+			Writes(spec.TfJob{}))
+
+	apiV1Ws.Route(
+		apiV1Ws.DELETE("/tfjob/{namespace}/{tfjob}").
+			To(apiHandler.handleDeleteTfJob))
+
+	apiV1Ws.Route(
+		apiV1Ws.GET("/logs/{namespace}/{podname}").
+			To(apiHandler.handleGetPodLogs).
+			Writes([]byte{}))
+
+	wsContainer.Add(apiV1Ws)
+	return wsContainer, nil
+}
+
+func (apiHandler *APIHandler) handleGetTfJobs(request *restful.Request, response *restful.Response) {
+
+	//TODO: namespace handling
+	jobs, err := apiHandler.cManager.TfJobClient.List("default")
+
+	if err != nil {
+		panic(err)
+	}
+
+	response.WriteHeaderAndEntity(http.StatusOK, jobs)
+}
+
+func (apiHandler *APIHandler) handleGetTfJobDetail(request *restful.Request, response *restful.Response) {
+	namespace := request.PathParameter("namespace")
+	name := request.PathParameter("tfjob")
+
+	job, err := apiHandler.cManager.TfJobClient.Get(namespace, name)
+	if err != nil {
+		panic(err)
+	}
+
+	tfJobDetail := TfJobDetail{
+		TfJob: job,
+	}
+
+	if job.Spec.TensorBoard != nil {
+		tbSpec, err := apiHandler.cManager.ClientSet.CoreV1().Services(namespace).List(metav1.ListOptions{
+			LabelSelector: fmt.Sprintf("tensorflow.org=,app=tensorboard,runtime_id=%s", job.Spec.RuntimeId),
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		if len(tbSpec.Items) > 0 {
+			// Should never be more than 1 service that matched, handle error
+			// Handle case where no tensorboard is found
+			tfJobDetail.TbService = &tbSpec.Items[0]
+		} else {
+			fmt.Println(fmt.Sprintf("Couldn't find a TensorBoard service for TfJob %s", job.Metadata.Name))
+		}
+	}
+
+	// Get associated pods
+	pods, err := apiHandler.cManager.ClientSet.CoreV1().Pods(namespace).List(metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("tensorflow.org=,runtime_id=%s", job.Spec.RuntimeId),
+	})
+	if err != nil {
+		panic(err)
+	}
+	tfJobDetail.Pods = pods.Items
+
+	response.WriteHeaderAndEntity(http.StatusOK, tfJobDetail)
+}
+
+func (apiHandler *APIHandler) handleDeploy(request *restful.Request, response *restful.Response) {
+	client := apiHandler.cManager.TfJobClient
+	spec := new(spec.TfJob)
+	if err := request.ReadEntity(spec); err != nil {
+		panic(err)
+	}
+	j, err := client.Create(spec.Metadata.Namespace, spec)
+	if err != nil {
+		panic(err)
+	}
+	response.WriteHeaderAndEntity(http.StatusCreated, j)
+}
+
+func (apiHandler *APIHandler) handleDeleteTfJob(request *restful.Request, response *restful.Response) {
+	namespace := request.PathParameter("namespace")
+	name := request.PathParameter("tfjob")
+	client := apiHandler.cManager.TfJobClient
+	err := client.Delete(namespace, name)
+	if err != nil {
+		panic(err)
+	}
+	response.WriteHeader(http.StatusOK)
+}
+
+func (apiHandler *APIHandler) handleGetPodLogs(request *restful.Request, response *restful.Response) {
+	namespace := request.PathParameter("namespace")
+	name := request.PathParameter("podname")
+
+	logs, err := apiHandler.cManager.ClientSet.CoreV1().Pods(namespace).GetLogs(name, &v1.PodLogOptions{}).Do().Raw()
+	if err != nil {
+		panic(err)
+	}
+
+	response.WriteHeaderAndEntity(http.StatusOK, string(logs))
+}

dashboard/backend/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/tensorflow/k8s/dashboard/backend/client"
+	"github.com/tensorflow/k8s/dashboard/backend/handler"
+)
+
+func main() {
+	log.SetOutput(os.Stdout)
+	cm, err := client.NewClientManager()
+	if err != nil {
+		log.Fatalf("Error while initializing connection to Kubernetes apiserver: %v", err)
+	}
+	apiHandler, err := handler.CreateHTTPAPIHandler(cm)
+	if err != nil {
+		log.Fatalf("Error while creating the API Handler: %v", err)
+	}
+
+	http.Handle("/api/", apiHandler)
+	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir("/opt/mlkube/dashboard/frontend/build/"))))
+
+	p := ":8080"
+	fmt.Println("Listening on", p)
+
+	http.ListenAndServe(p, nil)
+
+}

dashboard/frontend/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "tensorflowk8s-dashboard",
+  "version": "0.1.0",
+  "description": "Dashboard for tensorflow/k8s.",
+  "private": true,
+  "dependencies": {
+    "lodash": "^4.17.4",
+    "material-ui": "^0.19.4",
+    "react": "^16.0.0",
+    "react-dom": "^16.0.0",
+    "react-router-dom": "^4.2.2",
+    "react-scripts": "^1.0.17"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test --env=jsdom",
+    "eject": "react-scripts eject",
+    "toolbox": "react-toolbox-themr"
+  },
+  "license": "Apache-2.0"
+}

dashboard/frontend/public/index.html

@@ -0,0 +1,39 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <meta name="theme-color" content="#000000">
+    <!--
+      manifest.json provides metadata used when your web app is added to the
+      homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
+    -->
+    <link rel="shortcut icon" href="favicon.ico">
+    <!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+    <title>TensorFlow/k8s</title>
+  </head>
+  <body>
+    <noscript>
+      You need to enable JavaScript to run this app.
+    </noscript>
+    <div id="root"></div>
+    <!--
+      This HTML file is a template.
+      If you open it directly in the browser, you will see an empty page.
+
+      You can add webfonts, meta tags, or analytics to this file.
+      The build step will place the bundled scripts into the <body> tag.
+
+      To begin the development, run `npm start` or `yarn start`.
+      To create a production bundle, use `npm run build` or `yarn build`.
+    -->
+  </body>
+</html>

dashboard/frontend/src/components/App.css

@@ -0,0 +1,8 @@
+html {
+  font-family: roboto, sans-serif;
+}
+
+body {
+  font-family: 'Roboto', sans-serif;
+  background-color: #eee !important;
+}