etcdserver, client: propagate and print detailed error message

Current etcdserver just returns an error message "Internal Server
Error" in a case of internal error. This message isn't friendly and
not so useful for diagnosis. In addition, etcdctl just reports
"client: etcd cluster is unavailable or misconfigured" in such a case.

This commit improves the error message. The new body of error response
is now generated based on an error code of etcdserver. The client
constructs more friendly error message based on the response.

Below is an example:

Before:
 $ etcdctl member add infra6 http://127.0.0.1:32338
 client: etcd cluster is unavailable or misconfigured

After:
 $ etcdctl member add infra6 http://127.0.0.1:32338
 error #0: client: etcd member http://127.0.0.1:12379: etcdserver: re-configuration failed due to not enough started members
 error etcd-io#1: client: etcd member http://127.0.0.1:22379: etcdserver: re-configuration failed due to not enough started members
 error etcd-io#2: client: etcd member http://127.0.0.1:32379: etcdserver: re-configuration failed due to not enough started members

client/client.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"encoding/json"
 	"math/rand"
 	"net"
 	"net/http"
@@ -282,10 +283,21 @@ func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (*http.Respo
 			continue
 		}
 		if resp.StatusCode/100 == 5 {
+			type errMessage struct {
+				Message string
+			}
+
+			var msg errMessage
+			uerr := json.Unmarshal(body, &msg)
+			if uerr != nil {
+				// TODO: how to handle this case?
+				fmt.Errorf("json.Unmarshal() failed: %s", uerr)
+			}
+
 			switch resp.StatusCode {
 			case http.StatusInternalServerError, http.StatusServiceUnavailable:
 				// TODO: make sure this is a no leader response
-				cerr.Errors = append(cerr.Errors, fmt.Errorf("client: etcd member %s has no leader", eps[k].String()))
+				cerr.Errors = append(cerr.Errors, fmt.Errorf("client: etcd member %s: %s", eps[k].String(), msg.Message))
 			default:
 				cerr.Errors = append(cerr.Errors, fmt.Errorf("client: etcd member %s returns server error [%s]", eps[k].String(), http.StatusText(resp.StatusCode)))
 			}

etcdserver/etcdhttp/http.go

@@ -64,7 +64,7 @@ func writeError(w http.ResponseWriter, r *http.Request, err error) {
 		default:
 			plog.Errorf("got unexpected response error (%v)", err)
 		}
-		herr := httptypes.NewHTTPError(http.StatusInternalServerError, "Internal Server Error")
+		herr := httptypes.NewHTTPError(http.StatusInternalServerError, err.Error())
 		if et := herr.WriteTo(w); et != nil {
 			plog.Debugf("error writing HTTPError (%v) to %s", et, r.RemoteAddr)
 		}