[tools] CLI tool to simplify some tasks, and eliminate the curl commands from the docs

Makefile

@@ -93,6 +93,7 @@ TOOLS :=               \
 	verify_index_files   \
 	carbon_load          \
 	docs_test            \
+	m3ctl                \
 
 .PHONY: setup
 setup:
@@ -117,7 +118,7 @@ $(SERVICE)-linux-amd64:
 .PHONY: $(SERVICE)-docker-dev
 $(SERVICE)-docker-dev: clean-build $(SERVICE)-linux-amd64
 	mkdir -p ./bin/config
-	
+
 	# Hacky way to find all configs and put into ./bin/config/
 	find ./src | fgrep config | fgrep ".yml" | xargs -I{} cp {} ./bin/config/
 	find ./src | fgrep config | fgrep ".yaml" | xargs -I{} cp {} ./bin/config/

docs/operational_guide/placement_configuration.md

@@ -118,7 +118,7 @@ curl -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
             "endpoint": "<NODE_3_HOST_NAME>:<NODE_3_PORT>",
             "hostname": "<NODE_3_HOST_NAME>",
             "port": <NODE_3_PORT>
-        },
+        }
     ]
 }'
 ```

src/cmd/tools/m3ctl/README.md

@@ -0,0 +1,71 @@
+M3DB Tool
+========
+
+This is a CLI tool to do some things that may be desirable for
+cluster introspection, or for operational purposes.
+
+
+Where configuration data is required its provided via YAML.
+
+You can:
+
+* create a database per the simplified database create API
+* list namespaces
+* delete namespaces
+* list placements
+* delete placements
+* add nodes
+* remove nodes
+
+NOTE: This tool can delete namespaces and placements.  It can be
+quite hazardous if used without adequate understanding of your m3db
+cluster's topology, or without a decent understanding of how m3db
+works.
+
+Examples
+-------
+
+    # show help
+    m3ctl -h
+    # create a database
+    m3ctl db create -f ./database/examples/devel.yaml
+    # list namespaces
+    m3ctl ns
+    # delete a namespace
+    m3ctl ns delete -name default
+    # list placements
+    m3ctl pl
+    # point to some remote and list namespaces
+    m3ctl -endpoint http://localhost:7201 ns
+    # check the namespaces in a kubernetes cluster
+    # first setup a tunnel via kubectl port-forward ... 7201 
+    m3ctl -endpoint http://localhost:7201 ns
+    # list the ids of the placements
+    m3ctl -endpoint http://localhost:7201 pl | jq .placement.instances[].id
+
+Some example yaml files are provided in the examples directories.
+Here's one for database creation:
+
+    ---
+    type: cluster
+    namespace_name: default
+    retention_time: 168h
+    num_shards: 64
+    replication_factor: 1
+    hosts:
+    - id: m3db_seed
+      isolation_group: rack-a
+      zone: embedded
+      weight: 1024
+      endpoint: m3db_seed:9000
+      hostname: m3db_seed
+      port: 9000
+
+
+See the examples directories below.
+
+References
+==========
+
+[https://m3db.github.io/m3/operational_guide](operational guide)
+[api docs](https://www.m3db.io/openapi/)

src/cmd/tools/m3ctl/main/database/cmd.go

@@ -0,0 +1,95 @@
+package database
+
+import (
+	"flag"
+	"fmt"
+	"github.com/m3db/m3/src/cmd/tools/m3ctl/main/errors"
+	"github.com/m3db/m3/src/x/config/configflag"
+	"os"
+)
+
+const (
+	defaultPath = "/api/v1/database"
+)
+
+type DatabaseFlagSets struct {
+	Database *flag.FlagSet
+	Create   *flag.FlagSet
+}
+
+func SetupFlags(createDatabaseYAML *configflag.FlagStringSlice) DatabaseFlagSets {
+	databaseFlags := flag.NewFlagSet("db", flag.ExitOnError)
+	createFlags := flag.NewFlagSet("create", flag.ExitOnError)
+
+	databaseFlags.Usage = func() {
+		fmt.Fprintf(databaseFlags.Output(), `
+This is the "%s" subcommand for database scoped operations.
+
+It has the following subcommands:
+
+	%s
+
+`, databaseFlags.Name(), createFlags.Name())
+		databaseFlags.PrintDefaults()
+	}
+
+	createFlags.Var(createDatabaseYAML, "f", "Path to yaml for simplified db creation with sane defaults.")
+	createFlags.Usage = func() {
+		fmt.Fprintf(databaseFlags.Output(), `
+This is the "%s" subcommand for %s scoped operations.
+
+Description:
+
+This subcommand allows the creation of a new database from a yaml specification.
+
+Usage of %s:
+
+`, createFlags.Name(), databaseFlags.Name(), createFlags.Name())
+		createFlags.PrintDefaults()
+	}
+
+	return DatabaseFlagSets{Database: databaseFlags, Create: createFlags}
+}
+
+func ParseAndDo(arg *configflag.FlagStringSlice, flags *DatabaseFlagSets, ep string) {
+	args := flag.Args()
+	// right here args should be like "db create -f somefile"
+	if len(args) < 2 {
+		flags.Database.Usage()
+		os.Exit(1)
+	}
+	// pop and parse
+	if err := parseAndDoCreate(args[1:], arg, flags, ep, Create); err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+func parseAndDoCreate(args []string, finalArgs *configflag.FlagStringSlice, flags *DatabaseFlagSets, ep string, doer func(string, string)) error {
+	if err := flags.Database.Parse(args); err != nil {
+		flags.Database.Usage()
+		return err
+	}
+	if flags.Database.NArg() == 0 {
+		flags.Database.Usage()
+		return &errors.FlagsError{}
+	}
+	switch flags.Database.Arg(0) {
+	case flags.Create.Name():
+		// pop and parse
+		createArgs := flags.Database.Args()[1:]
+		if err := flags.Create.Parse(createArgs); err != nil {
+			flags.Create.Usage()
+			return err
+		}
+		if flags.Create.NFlag() == 0 {
+			flags.Create.Usage()
+			return &errors.FlagsError{}
+		}
+		// the below finalArgs.Value has at least one value by this time per the parser
+		doer(finalArgs.Value[len(finalArgs.Value)-1], ep)
+		return nil
+	default:
+		flags.Database.Usage()
+		return &errors.FlagsError{}
+	}
+}

src/cmd/tools/m3ctl/main/database/cmd_test.go

@@ -0,0 +1,32 @@
+package database
+
+import (
+	"github.com/m3db/m3/src/x/config/configflag"
+	"testing"
+)
+
+func TestBasic(t *testing.T) {
+
+	createDatabaseYAML := configflag.FlagStringSlice{}
+	databaseFlagSets := SetupFlags(&createDatabaseYAML)
+
+	if e := parseAndDoCreate([]string{}, &createDatabaseYAML, &databaseFlagSets, "", func(string, string) { return }); e == nil {
+		t.Error("It should return error on no args")
+	}
+
+	createDatabaseYAML = configflag.FlagStringSlice{}
+	databaseFlagSets = SetupFlags(&createDatabaseYAML)
+
+	if e := parseAndDoCreate([]string{"create", "-f", "somefile"}, &createDatabaseYAML, &databaseFlagSets, "", func(string, string) { return }); e != nil {
+		t.Error("It should NOT return error on sane args")
+	}
+
+	if len(createDatabaseYAML.Value) != 1 {
+		t.Fatalf("db create filename len is wrong:expected:1:but got:%d:\n", len(createDatabaseYAML.Value))
+	}
+
+	if createDatabaseYAML.Value[0] != "somefile" {
+		t.Errorf("db create filename value is wrong:expected:%s:but got:%s:\n", createDatabaseYAML.Value[0], "somefile")
+	}
+
+}

src/cmd/tools/m3ctl/main/database/create.go

@@ -0,0 +1,17 @@
+package database
+
+import (
+	"fmt"
+	"github.com/m3db/m3/src/cmd/tools/m3ctl/main/http"
+	"github.com/m3db/m3/src/cmd/tools/m3ctl/main/yaml"
+	"github.com/m3db/m3/src/query/generated/proto/admin"
+	"log"
+)
+
+func Create(createYAML string, endpoint string) {
+	log.Printf("createYAML:%s:\n", createYAML)
+	data := yaml.Load(createYAML, &admin.DatabaseCreateRequest{})
+	url := fmt.Sprintf("%s%s/create", endpoint, defaultPath)
+	http.DoPost(url, data, http.DoDump)
+	return
+}

src/cmd/tools/m3ctl/main/database/examples/create.yaml

@@ -0,0 +1,26 @@
+---
+type: cluster
+namespace_name: 1week_namespace
+retention_time: 168h
+num_shards: 1024
+replication_factor: 3
+hosts:
+- id: m3db001
+  isolationGroup: us-east1-a
+  zone: embedded
+  weight: 100
+  address: 10.142.0.1
+  port: 9000
+- id: m3db002
+  isolationGroup: us-east1-b
+  zone: embedded
+  weight: 100
+  address: 10.142.0.2
+  port: 9000
+- id: m3db003
+  isolationGroup: us-east1-c
+  zone: embedded
+  weight: 100
+  address: 10.142.0.3
+  port: 9000
+

src/cmd/tools/m3ctl/main/database/examples/devel.yaml

@@ -0,0 +1,15 @@
+---
+type: cluster
+namespace_name: default
+retention_time: 168h
+num_shards: 64
+replication_factor: 1
+hosts:
+- id: m3db_seed
+  isolation_group: rack-a
+  zone: embedded
+  weight: 1024
+  endpoint: m3db_seed:9000
+  hostname: m3db_seed
+  port: 9000
+

src/cmd/tools/m3ctl/main/errors/types.go

@@ -0,0 +1,12 @@
+package errors
+
+type FlagsError struct {
+	Message string
+}
+
+func (e *FlagsError) Error() string {
+	if e == nil {
+		return ""
+	}
+	return e.Message
+}

src/cmd/tools/m3ctl/main/http/checker.go

@@ -0,0 +1,19 @@
+package http
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+func checkForAndHandleError(url string, resp *http.Response) {
+	log.Printf("resp.StatusCode:%d:\n", resp.StatusCode)
+	if resp.StatusCode > 299 {
+		dat, _ := ioutil.ReadAll(resp.Body)
+		if dat != nil {
+			fmt.Println(string(dat))
+		}
+		log.Fatalf("error from m3db:%s:url:%s:", resp.Status, url)
+	}
+}

src/cmd/tools/m3ctl/main/http/http.go

@@ -0,0 +1,76 @@
+package http
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"time"
+)
+
+const timeout = time.Duration(5 * time.Second)
+
+func DoGet(url string, getter func(reader io.Reader)) {
+	log.Printf("DoGet:url:%s:\n", url)
+	client := http.Client{
+		Timeout: timeout,
+	}
+	resp, err := client.Get(url)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	checkForAndHandleError(url, resp)
+	getter(resp.Body)
+}
+
+func DoPost(url string, data io.Reader, getter func(reader io.Reader)) {
+	log.Printf("DoPost:url:%s:\n", url)
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	req, err := http.NewRequest(http.MethodPost, url, data)
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Fatal(err)
+
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	checkForAndHandleError(url, resp)
+	getter(resp.Body)
+}
+
+func DoDelete(url string, getter func(reader io.Reader)) {
+	log.Printf("DoDelete:url:%s:\n", url)
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	checkForAndHandleError(url, resp)
+	getter(resp.Body)
+}
+
+func DoDump(in io.Reader) {
+	dat, err := ioutil.ReadAll(in)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(dat))
+}