DNM: TiUP Cluster UI

Makefile

@@ -32,7 +32,7 @@ include ./tests/Makefile
 # Build TiUP and all components
 build: tiup components
 
-components: playground client cluster dm bench server
+components: playground client cluster dm bench web server 
 
 tiup:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup
@@ -58,6 +58,9 @@ doc:
 err:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-err ./components/err
 
+web:
+	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-web ./components/web
+
 server:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-server ./server
 

components/cluster/command/display.go

@@ -58,7 +58,7 @@ func newDisplayCmd() *cobra.Command {
 				return displayDashboardInfo(clusterName)
 			}
 
-			err = manager.Display(clusterName, gOpt)
+			_, err = manager.Display(clusterName, gOpt)
 			if err != nil {
 				return perrs.AddStack(err)
 			}

components/cluster/command/list.go

@@ -22,7 +22,8 @@ func newListCmd() *cobra.Command {
 		Use:   "list",
 		Short: "List all clusters",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return manager.ListCluster()
+			_, err := manager.ListCluster()
+			return err
 		},
 	}
 	return cmd

components/dm/command/display.go

@@ -40,7 +40,7 @@ func newDisplayCmd() *cobra.Command {
 
 			clusterName = args[0]
 
-			err := manager.Display(clusterName, gOpt)
+			_, err := manager.Display(clusterName, gOpt)
 			if err != nil {
 				return perrs.AddStack(err)
 			}

components/dm/command/list.go

@@ -22,7 +22,8 @@ func newListCmd() *cobra.Command {
 		Use:   "list",
 		Short: "List all clusters",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return manager.ListCluster()
+			_, err := manager.ListCluster()
+			return err
 		},
 	}
 	return cmd

components/web/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pingcap/tiup/pkg/cluster"
+	operator "github.com/pingcap/tiup/pkg/cluster/operation"
+	"github.com/pingcap/tiup/pkg/cluster/spec"
+	cors "github.com/rs/cors/wrapper/gin"
+)
+
+var tidbSpec *spec.SpecManager
+var manager *cluster.Manager
+
+func main() {
+	if err := spec.Initialize("cluster"); err != nil {
+		panic("initialize spec failed")
+	}
+	tidbSpec = spec.GetSpecManager()
+	manager = cluster.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion)
+
+	router := gin.Default()
+	router.Use(cors.AllowAll())
+	api := router.Group("/api")
+	{
+		api.GET("/clusters", clustersHandler)
+		api.GET("/clusters/:clusterName", clusterHandler)
+		api.DELETE("/clusters/:clusterName", destroyClusterHandler)
+
+		api.POST("/deploy", deployHandler)
+		api.GET("/deploy_status", deployStatusHandler)
+	}
+	_ = router.Run()
+}
+
+// DeployReq represents for the request of deploy API
+type DeployReq struct {
+	ClusterName string `json:"cluster_name"`
+	TiDBVersion string `json:"tidb_version"`
+	TopoYaml    string `json:"topo_yaml"`
+}
+
+func deployHandler(c *gin.Context) {
+	var req DeployReq
+	if err := c.ShouldBindJSON(&req); err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	// create temp topo yaml file
+	tmpfile, err := ioutil.TempFile("", "topo")
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	defer tmpfile.Close()
+	_, _ = tmpfile.WriteString(req.TopoYaml)
+	topoFilePath := tmpfile.Name()
+	fmt.Println("topo file path:", topoFilePath)
+
+	// parse request parameters
+	// topoFilePath = "/Users/baurine/Codes/Work/tiup/examples/manualTestEnv/multiHost/topology.yaml"
+	identifyFile := "/Users/baurine/Codes/Work/tiup/examples/manualTestEnv/_shared/vagrant_key"
+	go func() {
+		_ = manager.Deploy(
+			req.ClusterName,
+			req.TiDBVersion,
+			topoFilePath,
+			cluster.DeployOptions{
+				User:         "vagrant",
+				IdentityFile: identifyFile,
+			},
+			nil,
+			true,
+			120,
+			5,
+			false,
+		)
+	}()
+
+	c.JSON(http.StatusOK, gin.H{
+		"message": "ok",
+	})
+}
+
+func deployStatusHandler(c *gin.Context) {
+	status := manager.GetDeployStatus()
+	c.JSON(http.StatusOK, status)
+}
+
+func clustersHandler(c *gin.Context) {
+	clusters, err := manager.ListCluster()
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	c.JSON(http.StatusOK, clusters)
+}
+
+func clusterHandler(c *gin.Context) {
+	clusterName := c.Param("clusterName")
+	instInfos, err := manager.Display(clusterName, operator.Options{
+		SSHTimeout: 5,
+		OptTimeout: 120,
+		APITimeout: 300,
+	})
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+	c.JSON(http.StatusOK, instInfos)
+}
+
+func destroyClusterHandler(c *gin.Context) {
+	clusterName := c.Param("clusterName")
+	err := manager.DestroyCluster(clusterName, operator.Options{
+		SSHTimeout: 5,
+		OptTimeout: 120,
+		APITimeout: 300,
+	}, operator.Options{}, true)
+
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"message": "ok",
+	})
+}

examples/manualTestEnv/.gitignore

@@ -0,0 +1,2 @@
+.vagrant/
+tiup-cluster-*.log

examples/manualTestEnv/_shared/Vagrantfile.partial.pubKey.rb

@@ -0,0 +1,23 @@
+Vagrant.configure("2") do |config|
+  ssh_pub_key = File.readlines("#{File.dirname(__FILE__)}/vagrant_key.pub").first.strip
+
+  config.vm.box = "hashicorp/bionic64"
+  config.vm.provision "shell", privileged: false, inline: <<-SHELL
+    sudo apt install -y zsh
+    sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
+    sudo chsh -s /usr/bin/zsh vagrant
+    echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
+  SHELL
+
+  config.vm.provision "shell", privileged: true, inline: <<-SHELL
+    echo "setting ulimit"
+    sudo echo "fs.file-max = 65535" >> /etc/sysctl.conf
+    sudo sysctl -p
+    sudo echo "*               hard    nofile          65535" >> /etc/security/limits.conf
+    sudo echo "*               soft    nofile          65535" >> /etc/security/limits.conf
+    sudo echo "root            hard    nofile          65535" >> /etc/security/limits.conf
+    sudo echo "root            hard    nofile          65535" >> /etc/security/limits.conf
+  SHELL
+end
+
+# ulimit ref: https://my.oschina.net/u/914655/blog/3067520

examples/manualTestEnv/_shared/vagrant_key

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAxboZzYumqNoVOQ/hKKhIZHxNhf5tmnkLZry8i6Xur4FPLDiRxos/
+xVVDx0ynTPOyQVVaXtNxZnAmbR4HuNBzRvNoklwSXazt5YgWeiKCHtPpKFt3PJeE2cn6FJ
+p6F6qFChG0NSPbZxJWWxv4noX0U3PLKgHNIehYK2Fu0E6plhSZazzJEVWapwo9d7aGnAsz
+bBCd5TNZ5ogrXn+3bSFcdCbAfWOwYg54a+PzTQlzgt6JmhlEjpFfPhhpBW92pQXxmQ2c17
+iPCbA8G++FiaEwA5teex8k1+HzmHf7YjyhPr+I67EzEiIueJg2+0PYbM1p06S8kVTNDXsf
+0eJx4Dr8qQAAA9iFPcpVhT3KVQAAAAdzc2gtcnNhAAABAQDFuhnNi6ao2hU5D+EoqEhkfE
+2F/m2aeQtmvLyLpe6vgU8sOJHGiz/FVUPHTKdM87JBVVpe03FmcCZtHge40HNG82iSXBJd
+rO3liBZ6IoIe0+koW3c8l4TZyfoUmnoXqoUKEbQ1I9tnElZbG/iehfRTc8sqAc0h6FgrYW
+7QTqmWFJlrPMkRVZqnCj13toacCzNsEJ3lM1nmiCtef7dtIVx0JsB9Y7BiDnhr4/NNCXOC
+3omaGUSOkV8+GGkFb3alBfGZDZzXuI8JsDwb74WJoTADm157HyTX4fOYd/tiPKE+v4jrsT
+MSIi54mDb7Q9hszWnTpLyRVM0Nex/R4nHgOvypAAAAAwEAAQAAAQBtk0+/YDgQ9SKzx8AQ
+xwmvXk+cBT76T0BpRAj9HwziiDe3GvZ2YC8MDc+NAEbq11ae7E0zpdv/WAGDkRPYcPShij
+0Wdx3aef4wqLVEJCGWMfvRWLcAhjuiclM73cvxl5c42EzU8jUhrsDapuql9zhKky4w7mSe
++OL7z3gYyq8isvcQMe+1eXJqiv27AJJfAir+rLJZO/gDW36hOowhnZxYRlVYPgZ8GwetxD
+VdCrgwUgR/2HYmbXYdVxI0PwswGc6rEqs5XXOYRzwvPTvRKdD3J5MxmsvJljT7FMr4kCLT
+X1+aWysk1cgAUIdzzwQL8DLE/N9PFFYdZyNBkZMgedl9AAAAgCtP3F8XYFR18gQLPGLDyQ
+FFg8+JHN9b/yIg2pymC6SI8qEp+GnuEK9IKhqh/Uw14KEKcs/9sgbZo0K9uTBTDG5F6Qmp
+hADVbWXJ/97Xeya6kH2Sa56UKLCQ/uQWBKwLQ0auU/qwxATIZowh31XUXjzVBg6wgUjT7Q
++3Fk1zGYxnAAAAgQD5USIRUNwkI+htv+f1g8QdmrFAGymcGEkXAixKvBTon9cWQb2iyiK+
+2IO8EwFwRdL5kw2foILCnlp/4FevfxHU7wTcoFEp3PItUlcxYqO8vY2VCZ913oNLKBIt9p
+uFfG2BZM5szMRNMh0svelu61FePsfN5Z8J0ltPrS8UKB95ywAAAIEAywbyNbjz1AxEjWIX
+2Vbk4/MjQyjui8Wi7H0F+LDWyMfPJHzhnbr79Z/lIZmDAo++3EYU9J9s0C+wJ6vXGK+gvC
+7e5qGfT/0J0DwBfLbpeTdDELCa/LmfLWVPzZ9Q+9Fq0AjmW9YXFZ/+qT9xfY1v9XfztFRS
+xR1iXJ42q6ff5NsAAAAeYnJlZXpld2lzaEBCcmVlemV3aXNoTUJQLmxvY2FsAQIDBAU=
+-----END OPENSSH PRIVATE KEY-----

examples/manualTestEnv/_shared/vagrant_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFuhnNi6ao2hU5D+EoqEhkfE2F/m2aeQtmvLyLpe6vgU8sOJHGiz/FVUPHTKdM87JBVVpe03FmcCZtHge40HNG82iSXBJdrO3liBZ6IoIe0+koW3c8l4TZyfoUmnoXqoUKEbQ1I9tnElZbG/iehfRTc8sqAc0h6FgrYW7QTqmWFJlrPMkRVZqnCj13toacCzNsEJ3lM1nmiCtef7dtIVx0JsB9Y7BiDnhr4/NNCXOC3omaGUSOkV8+GGkFb3alBfGZDZzXuI8JsDwb74WJoTADm157HyTX4fOYd/tiPKE+v4jrsTMSIi54mDb7Q9hszWnTpLyRVM0Nex/R4nHgOvyp

examples/manualTestEnv/multiHost/README.md

@@ -0,0 +1,36 @@
+# multiHost
+
+TiDB, PD, TiKV, TiFlash each in different hosts.
+
+## Usage
+
+1. Start the box:
+
+   ```bash
+   vagrant up
+   ```
+
+1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once):
+
+   ```bash
+   tiup cluster deploy multiHost v4.0.4 topology.yaml -i ../_shared/vagrant_key -y --user vagrant
+   ```
+
+1. Start the cluster in the box:
+
+   ```bash
+   tiup cluster start multiHost
+   ```
+
+1. Start TiDB Dashboard server:
+
+   ```bash
+   bin/tidb-dashboard --pd http://10.0.1.11:2379
+   ```
+
+## Cleanup
+
+```bash
+tiup cluster destroy multiHost -y
+vagrant destroy --force
+```

examples/manualTestEnv/multiHost/Vagrantfile

@@ -0,0 +1,14 @@
+load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.rb"
+
+Vagrant.configure("2") do |config|
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 1024
+    v.cpus = 1
+  end
+
+  (1..3).each do |i|
+    config.vm.define "node#{i}" do |node|
+      node.vm.network "private_network", ip: "10.0.1.#{i+10}"
+    end
+  end
+end

examples/manualTestEnv/multiHost/topology.yaml

@@ -0,0 +1,42 @@
+global:
+  user: tidb
+  deploy_dir: tidb-deploy
+  data_dir: tidb-data
+
+server_configs:
+  tikv:
+    server.grpc-concurrency: 1
+    raftstore.apply-pool-size: 1
+    raftstore.store-pool-size: 1
+    readpool.unified.max-thread-count: 1
+    readpool.storage.use-unified-pool: false
+    readpool.coprocessor.use-unified-pool: true
+    storage.block-cache.capacity: 256MB
+    raftstore.capacity: 10GB
+  pd:
+    replication.enable-placement-rules: true
+
+pd_servers:
+  - host: 10.0.1.11
+  - host: 10.0.1.12
+  - host: 10.0.1.13
+
+tikv_servers:
+  - host: 10.0.1.12
+
+tidb_servers:
+  - host: 10.0.1.11
+  - host: 10.0.1.12
+  - host: 10.0.1.13
+
+# tiflash_servers:
+#   - host: 10.0.1.14
+
+grafana_servers:
+  - host: 10.0.1.11
+
+monitoring_servers:
+  - host: 10.0.1.11
+
+alertmanager_servers:
+  - host: 10.0.1.11

examples/manualTestEnv/multiReplica/README.md

@@ -0,0 +1,36 @@
+# multiReplica
+
+Multiple TiKV nodes in different labels.
+
+## Usage
+
+1. Start the box:
+
+   ```bash
+   vagrant up
+   ```
+
+1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once):
+
+   ```bash
+   tiup cluster deploy multiReplica v4.0.4 topology.yaml -i ../_shared/vagrant_key -y --user vagrant
+   ```
+
+1. Start the cluster in the box:
+
+   ```bash
+   tiup cluster start multiReplica
+   ```
+
+1. Start TiDB Dashboard server:
+
+   ```bash
+   bin/tidb-dashboard --pd http://10.0.1.20:2379
+   ```
+
+## Cleanup
+
+```bash
+tiup cluster destroy multiReplica -y
+vagrant destroy --force
+```

examples/manualTestEnv/multiReplica/Vagrantfile

@@ -0,0 +1,10 @@
+load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.rb"
+
+Vagrant.configure("2") do |config|
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 4 * 1024
+    v.cpus = 2
+  end
+
+  config.vm.network "private_network", ip: "10.0.1.20"
+end