[tools] CLI tool to simplify some tasks, and eliminate the curl commands from the docs

Makefile

@@ -95,6 +95,7 @@ TOOLS :=               \
 	verify_index_files   \
 	carbon_load          \
 	docs_test            \
+	m3ctl                \
 
 .PHONY: setup
 setup:

docs/operational_guide/placement_configuration.md

@@ -118,7 +118,7 @@ curl -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
             "endpoint": "<NODE_3_HOST_NAME>:<NODE_3_PORT>",
             "hostname": "<NODE_3_HOST_NAME>",
             "port": <NODE_3_PORT>
-        },
+        }
     ]
 }'
 ```

src/cmd/tools/m3ctl/README.md

@@ -0,0 +1,82 @@
+M3DB CLI Tool
+=============
+
+This is a CLI tool to do some things that may be desirable for
+cluster introspection, or for operational purposes.
+
+Where configuration data is required its provided via YAML.
+
+You can:
+
+* create a database per the simplified database create API
+* list namespaces
+* delete namespaces
+* list placements
+* delete placements
+* add nodes
+* remove nodes
+
+NOTE: This tool can delete namespaces and placements.  It can be
+quite hazardous if used without adequate understanding of your m3db
+cluster's topology, or without a decent understanding of how m3db
+works.
+
+Examples
+-------
+
+    # show help
+    m3ctl -h
+    # create a database
+    m3ctl apply -f ./database/examples/dbcreate.yaml
+    # list namespaces
+    m3ctl get ns
+    # delete a namespace
+    m3ctl delete ns -id default
+    # list placements
+    m3ctl get pl
+    # point to some remote and list namespaces
+    m3ctl -endpoint http://localhost:7201 get ns
+    # check the namespaces in a kubernetes cluster
+    # first setup a tunnel via kubectl port-forward ... 7201 
+    m3ctl -endpoint http://localhost:7201 get ns
+    # list the ids of the placements
+    m3ctl -endpoint http://localhost:7201 get pl | jq .placement.instances[].id
+
+Some example yaml files for the "apply" subcommand are provided in the yaml/examples directory.
+Here's one to initialize a topology:
+
+    ---
+    operation: init
+    num_shards: 64
+    replication_factor: 1
+    instances:
+      - id: nodeid1
+        isolation_group: isogroup1
+        zone: etcd1
+        weight: 100
+        endpoint: node1:9000
+        hostname: node1
+        port: 9000
+      - id: nodeid2
+        isolation_group: isogroup2
+        zone: etcd1
+        weight: 100
+        endpoint: node2:9000
+        hostname: node2
+        port: 9000
+      - id: nodeid3
+        isolation_group: isogroup3
+        zone: etcd1
+        weight: 100
+        endpoint: node3:9000
+        hostname: node3
+        port: 9000
+
+
+See the examples directories below.
+
+References
+==========
+
+[https://m3db.github.io/m3/operational_guide](operational guide)
+[api docs](https://www.m3db.io/openapi/)

src/cmd/tools/m3ctl/apply/apply.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package apply
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+
+	"github.com/m3db/m3/src/cmd/tools/m3ctl/client"
+	"github.com/m3db/m3/src/cmd/tools/m3ctl/yaml"
+)
+
+// DoApply does the API call to handle the kubectl apply command
+// It looks at the yaml, figures out what kind of API call to make
+// Sends it all off to the API
+func DoApply(endpoint string, filepath string, logger *zap.Logger) error {
+	path, data, err := yaml.Load(filepath, logger)
+	if err != nil {
+		return err
+	}
+	url := fmt.Sprintf("%s%s", endpoint, path)
+	return client.DoPost(url, data, client.Dumper, logger)
+}

src/cmd/tools/m3ctl/client/checker.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package client
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"go.uber.org/zap"
+)
+
+func checkForAndHandleError(url string, resp *http.Response, zl *zap.Logger) error {
+	if resp.StatusCode/100 != 2 {
+		dat, _ := ioutil.ReadAll(resp.Body)
+		if dat != nil {
+			zl.Error("error response",
+				zap.Error(fmt.Errorf("status %d", resp.StatusCode)),
+				zap.String("url", url),
+				zap.ByteString("response", dat))
+		} else {
+			zl.Error("error response",
+				zap.Error(fmt.Errorf("status %d", resp.StatusCode)),
+				zap.String("url", url),
+				zap.ByteString("response", dat))
+		}
+		return fmt.Errorf("error from m3db status=%s, url=%s\n", resp.Status, url)
+	}
+	return nil
+}

src/cmd/tools/m3ctl/client/http.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package client
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+const timeout = time.Duration(5 * time.Second)
+
+// DoGet is the low level call to the backend api for gets
+func DoGet(url string, getter func(reader io.Reader, zl *zap.Logger) error, zl *zap.Logger) error {
+	zl.Info("request", zap.String("method", "get"), zap.String("url", url))
+	client := http.Client{
+		Timeout: timeout,
+	}
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	if err := checkForAndHandleError(url, resp, zl); err != nil {
+		return err
+	}
+	return getter(resp.Body, zl)
+}
+
+// DoPost is the low level call to the backend api for posts
+func DoPost(url string, data io.Reader, getter func(reader io.Reader, zl *zap.Logger) error, zl *zap.Logger) error {
+	zl.Info("request", zap.String("method", "post"), zap.String("url", url))
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	req, err := http.NewRequest(http.MethodPost, url, data)
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	if err := checkForAndHandleError(url, resp, zl); err != nil {
+		return err
+	}
+	return getter(resp.Body, zl)
+}
+
+// DoDelete is the low level call to the backend api for deletes
+func DoDelete(url string, getter func(reader io.Reader, zl *zap.Logger) error, zl *zap.Logger) error {
+	zl.Info("request", zap.String("method", "delete"), zap.String("url", url))
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+	if err := checkForAndHandleError(url, resp, zl); err != nil {
+		return err
+	}
+	return getter(resp.Body, zl)
+}
+
+// Dumper is a simple printer for http responses
+func Dumper(in io.Reader, zl *zap.Logger) error {
+	dat, err := ioutil.ReadAll(in)
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(dat))
+	return nil
+}

src/cmd/tools/m3ctl/errors/types.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package errors
+
+type FlagsError struct {
+	Message string
+}
+
+func (e *FlagsError) Error() string {
+	if e == nil {
+		return ""
+	}
+	return e.Message
+}