[api] Implement replace endpoint (#1162)

Adds an endpoint for replacing an instance across all services.

.gitignore

@@ -10,6 +10,7 @@
 *.cov
 *.html
 *.tmp
+.DS_Store
 test.log
 
 # glide manages this
@@ -46,4 +47,3 @@ site/
 !m3db.io/**/vendor
 # Automatically populated from asset sources
 m3db.io/openapi
-

README.md

@@ -90,15 +90,15 @@ produce a lightweight production image from a single Dockerfile. Accordingly, it
 17.05 or later to build.
 
 ```
-docker build -t m3dbnode:$(git rev-parse head) .
+docker build -f docker/m3dbnode/Dockerfile -t m3dbnode:$(git rev-parse head) .
 docker run --name m3dbnode m3dbnode:$(git rev-parse head)
 ```
 
 If you wish to build an image with the source code included you can stop the build after the
 `builder` stage:
 
 ```
-docker build -t m3dbnode:$(git rev-parse head) --target builder .
+docker build -f docker/m3dbnode/Dockerfile -t m3dbnode:$(git rev-parse head) --target builder .
 ```
 
 ## Configuration

docs/operational_guide/placement_configuration.md

@@ -64,10 +64,10 @@ The instructions below all contain sample curl commands, but you can always revi
 
 #### Placement Initialization
 
-Send a POST request to the `/api/v1/placement/init` endpoint
+Send a POST request to the `/api/v1/services/m3db/placement/init` endpoint
 
 ```bash
-curl -X POST localhost:7201/api/v1/placement/init -d '{
+curl -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
     "num_shards": <DESIRED_NUMBER_OF_SHARDS>,
     "replication_factor": <DESIRED_REPLICATION_FACTOR>(recommended 3),
     "instances": [
@@ -104,10 +104,10 @@ curl -X POST localhost:7201/api/v1/placement/init -d '{
 
 #### Adding a Node
 
-Send a POST request to the `/api/v1/placement` endpoint
+Send a POST request to the `/api/v1/services/m3db/placement` endpoint
 
 ```bash
-curl -X POST <M3_COORDINATOR_HOST_NAME>:<M3_COORDINATOR_PORT(default 7201)>/api/v1/placement -d '{
+curl -X POST <M3_COORDINATOR_HOST_NAME>:<M3_COORDINATOR_PORT(default 7201)>/api/v1/services/m3db/placement -d '{
   "instances": [
     {
       "id": "<NEW_NODE_ID>",
@@ -126,16 +126,31 @@ After sending the add command you will need to wait for the M3DB cluster to reac
 
 #### Removing a Node
 
-Send a DELETE request to the `/api/v1/placement/<NODE_ID>` endpoint.
+Send a DELETE request to the `/api/v1/services/m3db/placement/<NODE_ID>` endpoint.
 
 ```bash
-curl -X DELETE <M3_COORDINATOR_HOST_NAME>:<M3_COORDINATOR_PORT(default 7201)>/api/v1/placement/<NODE_ID>
+curl -X DELETE <M3_COORDINATOR_HOST_NAME>:<M3_COORDINATOR_PORT(default 7201)>/api/v1/services/m3db/placement/<NODE_ID>
 ```
 
 After sending the delete command you will need to wait for the M3DB cluster to reach the new desired state. You'll know that this has been achieved when the placement shows that all shards for all hosts are in the `Available` state.
 
 #### Replacing a Node
 
-Currently, the best way to replace a node (due to hardware failure or any other reason) is to perform a node remove followed by a node add. We will eventually support node replacement as a single operation, but that is currently not implemented.
-
+Send a POST request to the `/api/v1/services/m3db/placement/replace` endpoint containing hosts to replace and candidates to replace it with.
 
+```bash
+curl -X POST <M3_COORDINATOR_HOST_NAME>:<M3_COORDINATOR_PORT(default 7201)>/api/v1/services/m3db/placement/replace -d '{
+    "leavingInstanceIDs": ["<OLD_NODE_ID>"],
+    "candidates": [
+        {
+          "id": "<NEW_NODE_ID>",
+          "isolationGroup": "<NEW_NODE_ISOLATION_GROUP>",
+          "zone": "<ETCD_ZONE>",
+          "weight": <NODE_WEIGHT>,
+          "endpoint": "<NEW_NODE_HOST_NAME>:<NEW_NODE_PORT>(default 9000)",
+          "hostname": "<NEW_NODE_HOST_NAME>",
+          "port": <NEW_NODE_PORT>
+        }
+    ]
+}'
+```

src/query/api/v1/handler/placement/add.go

@@ -21,7 +21,6 @@
 package placement
 
 import (
-	"fmt"
 	"net/http"
 	"path"
 	"time"
@@ -62,14 +61,6 @@ var (
 // AddHandler is the handler for placement adds.
 type AddHandler Handler
 
-type unsafeAddError struct {
-	hosts string
-}
-
-func (e unsafeAddError) Error() string {
-	return fmt.Sprintf("instances [%s] do not have all shards available", e.hosts)
-}
-
 // NewAddHandler returns a new instance of AddHandler.
 func NewAddHandler(opts HandlerOptions) *AddHandler {
 	return &AddHandler{HandlerOptions: opts, nowFn: time.Now}
@@ -168,5 +159,6 @@ func (h *AddHandler) Add(
 		return nil, err
 	}
 
+	// TODO(schallert): change after https://github.com/m3db/m3/issues/1165
 	return newPlacement.SetVersion(version + 1), nil
 }

src/query/api/v1/handler/placement/add_test.go

@@ -154,7 +154,7 @@ func TestPlacementAddHandler_SafeOK(t *testing.T) {
 		)
 		switch serviceName {
 		case M3AggregatorServiceName:
-			req = httptest.NewRequest(AddHTTPMethod, M3DBAddURL, strings.NewReader(`{"instances":[{"id": "host1","isolation_group": "rack1","zone": "test","weight": 1,"endpoint": "http://host1:1234","hostname": "host1","port": 1234}]}`))
+			req = httptest.NewRequest(AddHTTPMethod, M3AggAddURL, strings.NewReader(`{"instances":[{"id": "host1","isolation_group": "rack1","zone": "test","weight": 1,"endpoint": "http://host1:1234","hostname": "host1","port": 1234}]}`))
 		default:
 			req = httptest.NewRequest(AddHTTPMethod, M3DBAddURL, strings.NewReader(`{"instances":[{"id": "host1","isolation_group": "rack1","zone": "test","weight": 1,"endpoint": "http://host1:1234","hostname": "host1","port": 1234}]}`))
 		}

src/query/api/v1/handler/placement/common.go

@@ -381,6 +381,15 @@ func RegisterRoutes(r *mux.Router, opts HandlerOptions) {
 	r.HandleFunc(M3DBDeleteURL, deleteFn).Methods(DeleteHTTPMethod)
 	r.HandleFunc(M3AggDeleteURL, deleteFn).Methods(DeleteHTTPMethod)
 	r.HandleFunc(M3CoordinatorDeleteURL, getFn).Methods(GetHTTPMethod)
+
+	// Replace
+	var (
+		replaceHandler = NewReplaceHandler(opts)
+		replaceFn      = applyMiddleware(replaceHandler.ServeHTTP)
+	)
+	r.HandleFunc(M3DBReplaceURL, replaceFn).Methods(ReplaceHTTPMethod)
+	r.HandleFunc(M3AggReplaceURL, replaceFn).Methods(ReplaceHTTPMethod)
+	r.HandleFunc(M3CoordinatorReplaceURL, replaceFn).Methods(ReplaceHTTPMethod)
 }
 
 func newPlacementCutoverNanosFn(
@@ -460,6 +469,14 @@ type m3aggregatorPlacementOpts struct {
 	propagationDelay time.Duration
 }
 
+type unsafeAddError struct {
+	hosts string
+}
+
+func (e unsafeAddError) Error() string {
+	return fmt.Sprintf("instances [%s] do not have all shards available", e.hosts)
+}
+
 func validateAllAvailable(p placement.Placement) error {
 	badInsts := []string{}
 	for _, inst := range p.Instances() {
@@ -517,3 +534,11 @@ func parseServiceFromRequest(r *http.Request) (string, error) {
 
 	return "", errUnableToParseService
 }
+
+func isStateless(serviceName string) bool {
+	switch serviceName {
+	case M3CoordinatorServiceName:
+		return true
+	}
+	return false
+}