Merge pull request #1 from qvantel/usability-improvements

Improved user/dev usability

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Found an issue with nerd? Let us know so we can fix it
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Environment**
+* nerd:
+  * Version:
+  * Config (retrieve with `docker inspect nerd --format='{{json .Config.Env}}'` or equivalent):
+    ```json
+    # Paste your nerd environment variables here.
+    # Be sure to scrub any sensitive values
+    ```
+* Kafka:
+  * Version:
+  * Config  (retrieve with `grep -v -e "^$" -e "#" server.properties` from the Kafka config dir or equivalent):
+    ```
+    # Paste your Kafka config here.
+    # Be sure to scrub any sensitive values
+    ```
+* Redis (if present):
+  * Version:
+  * Config (retrieve with `CONFIG GET *` from a redis-cli prompt):
+    ```
+    # Paste your Redis config here.
+    # Be sure to scrub any sensitive values
+    ```
+* Elasticsearch (if present):
+  * Version:
+  * Config (retrieve from `<elasticsearch-host>:9200/_cluster/settings?include_defaults`):
+    ```json
+    # Paste your Elasticsearch config here.
+    # Be sure to scrub any sensitive values
+    ```
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest something!
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Explain any additional use cases**
+If there are any use cases that would help us understand the use/need/value please share them as they can help us decide
+on acceptance and prioritization.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

Dockerfile

@@ -3,13 +3,17 @@ FROM golang:1.15.6-alpine3.12 AS build-go
 RUN apk --no-cache add git
 ENV D=/go/src/github.com/qvantel/nerd
 ADD ./ $D/
-RUN cd $D && CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' ./cmd/nerd/ && cp nerd /tmp/
+WORKDIR $D
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' ./cmd/nerd/ && \
+    cp nerd /tmp/
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' ./cmd/fcollect/ && \
+    cp fcollect /tmp/
 
 # 2) BUILD FINAL IMAGE
 FROM scratch
 WORKDIR /opt/docker
-COPY --from=build-go /tmp/nerd /opt/docker/
+COPY --from=build-go /tmp/nerd /tmp/fcollect /opt/docker/
 ENV GIN_MODE=release \
-    VERSION=0.2.1
+    VERSION=0.3.0
 EXPOSE 5400
 ENTRYPOINT [ "./nerd" ]

api/api.go

@@ -1,6 +1,9 @@
+// Package api contains the handlers and types that support nerd's REST API
 package api
 
 import (
+	"net/http"
+
 	"github.com/gin-gonic/gin"
 	"github.com/qvantel/nerd/api/types"
 	"github.com/qvantel/nerd/internal/config"
@@ -56,6 +59,7 @@ func New(tServ chan types.TrainRequest, conf config.Config) (*Handler, error) {
 	router.Use(gin.Recovery())
 
 	// Routes
+	router.GET("/", h.ShowWelcomeMsg)
 	v1 := router.Group("/api/v1")
 	{
 		health := v1.Group("/health")
@@ -74,10 +78,25 @@ func New(tServ chan types.TrainRequest, conf config.Config) (*Handler, error) {
 			series.GET("", h.ListSeries)
 			series.DELETE("/:id", h.DeleteSeries)
 			series.GET("/:id/points", h.ListPoints)
-			series.POST("/process", h.AddPoint)
+			series.POST("/process", h.ProcessEvent)
 		}
 	}
 
 	logger.Info("API initialized")
 	return &h, nil
 }
+
+// ShowWelcomeMsg is a simple handler method for showing a helpful message at the root of the HTTP server
+func (h *Handler) ShowWelcomeMsg(c *gin.Context) {
+	msg := `<!doctype html>
+<html>
+	<body>
+		<h1>nerd ` + h.Conf.AppVersion + `</h1>
+
+		Welcome to the nerd API! This is a restful machine learning service, if you'd like to learn more about it, maybe
+		check out the github project <a href="https://github.com/qvantel/nerd">here</a>.
+	</body>
+</html>`
+	c.Writer.WriteHeader(http.StatusOK)
+	c.Writer.Write([]byte(msg))
+}

api/health.go

@@ -4,14 +4,15 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"github.com/qvantel/nerd/api/types"
 )
 
 // StartupCheck godoc
 // @Summary Kubernetes startup probe endpoint
 // @Description Will return a 200 as long as the API is up
 // @Produce plain
-// @Success 200 {string} string
+// @Success 200 {object} types.SimpleRes
 // @Router /health/startup [get]
 func StartupCheck(c *gin.Context) {
-	c.String(http.StatusOK, "UP")
+	c.JSON(http.StatusOK, types.NewOkRes("The API is up"))
 }

api/nets.go

@@ -16,18 +16,18 @@ import (
 // @Description Will delete the net with the specified ID
 // @Produce json
 // @Param id path string true "Net ID"
-// @Success 200
-// @Failure 500 {object} types.APIError "When there is an error deleting the net"
+// @Success 200 {object} types.SimpleRes
+// @Failure 500 {object} types.SimpleRes "When there is an error deleting the net"
 // @Router /nets/{id} [delete]
 func (h *Handler) DeleteNet(c *gin.Context) {
 	id := c.Param("id")
 	err := h.MLS.Delete(id)
 	if err != nil {
 		logger.Error("Failed to delete net "+id, err)
-		c.JSON(http.StatusInternalServerError, types.APIError{Msg: "Error deleting net, see logs for more info"})
+		c.JSON(http.StatusInternalServerError, types.NewErrorRes("Error deleting net "+id+", see logs for more info"))
 		return
 	}
-	c.JSON(http.StatusOK, "")
+	c.JSON(http.StatusOK, types.NewOkRes("Net "+id+" was successfully deleted"))
 }
 
 // Evaluate godoc
@@ -37,43 +37,41 @@ func (h *Handler) DeleteNet(c *gin.Context) {
 // @Produce json
 // @Param id path string true "Net ID"
 // @Success 200 {object} map[string]float32
-// @Failure 400 {object} types.APIError "When the request body is formatted incorrectly"
-// @Failure 404 {object} types.APIError "When the provided net ID isn't found"
-// @Failure 500 {object} types.APIError "When there is an error loading the net or evaluating the inputs"
+// @Failure 400 {object} types.SimpleRes "When the request body is formatted incorrectly"
+// @Failure 404 {object} types.SimpleRes "When the provided net ID isn't found"
+// @Failure 500 {object} types.SimpleRes "When there is an error loading the net or evaluating the inputs"
 // @Router /nets/{id}/evaluate [post]
 func (h *Handler) Evaluate(c *gin.Context) {
 	id := c.Param("id")
 	var inputs map[string]float32
 	err := c.ShouldBind(&inputs)
 	if err != nil {
 		logger.Debug("Failed to unmarshal message (" + err.Error() + ")")
-		c.JSON(http.StatusBadRequest, types.APIError{Msg: "Wrong format"})
+		c.JSON(http.StatusBadRequest, types.NewErrorRes("Wrong format"))
 		return
 	}
 
 	_, err = ml.ID2Type(id)
 	if err != nil {
-		c.JSON(http.StatusBadRequest, types.APIError{Msg: err.Error()})
+		c.JSON(http.StatusBadRequest, types.NewErrorRes(err.Error()))
 		return
 	}
 
 	// The net has to exist for this to work so it's more memory efficient to pass the zero value of each type for the
 	// creation params
 	net, err := ml.NewNetwork(id, nil, nil, 0, false, h.MLS, h.Conf)
-	loadErr := types.APIError{Msg: "Error loading net, see logs for more info"}
 	if err != nil {
 		logger.Error("Failed to load net "+id, err)
-		c.JSON(http.StatusInternalServerError, loadErr)
+		c.JSON(http.StatusInternalServerError, types.NewErrorRes("Error loading net, see logs for more info"))
 		return
 	}
 	if net == nil {
-		c.JSON(http.StatusNotFound, types.APIError{Msg: "Net with id " + id + " could not be found"})
+		c.JSON(http.StatusNotFound, types.NewErrorRes("Net with ID "+id+" could not be found"))
 		return
 	}
 	res, err := net.Evaluate(inputs)
 	if err != nil {
-		logger.Error("Failed to evaluate inputs with net "+id, err)
-		c.JSON(http.StatusInternalServerError, types.APIError{Msg: "Error evaluting inputs, see logs for more info"})
+		c.JSON(http.StatusInternalServerError, types.NewErrorRes("Error evaluting inputs ("+err.Error()+")"))
 		return
 	}
 	c.JSON(http.StatusOK, res)
@@ -86,19 +84,19 @@ func (h *Handler) Evaluate(c *gin.Context) {
 // @Param offset query int false "Offset to fetch" default(0)
 // @Param limit query int false "How many networks to fetch, the service might return more in some cases" default(10) maximum(50)
 // @Success 200 {object} types.PagedRes
-// @Failure 400 {object} types.APIError "When the request params are formatted incorrectly"
-// @Failure 500 {object} types.APIError "When there is an error retrieving the list of nets"
+// @Failure 400 {object} types.SimpleRes "When the request params are formatted incorrectly"
+// @Failure 500 {object} types.SimpleRes "When there is an error retrieving the list of nets"
 // @Router /nets [get]
 func (h *Handler) ListNets(c *gin.Context) {
 	raw := c.DefaultQuery("offset", "0")
 	offset, err := strconv.Atoi(raw)
 	if err != nil {
-		c.JSON(http.StatusBadRequest, types.APIError{Msg: "offset must be a valid integer"})
+		c.JSON(http.StatusBadRequest, types.NewErrorRes("offset must be a valid integer"))
 	}
 	raw = c.DefaultQuery("limit", "10")
 	limit, err := strconv.Atoi(raw)
 	if err != nil {
-		c.JSON(http.StatusBadRequest, types.APIError{Msg: "limit must be a valid integer"})
+		c.JSON(http.StatusBadRequest, types.NewErrorRes("limit must be a valid integer"))
 	}
 	if limit > 50 {
 		limit = 50 // Till there is a better solution in place, this is so things won't get too much out of control
@@ -107,7 +105,7 @@ func (h *Handler) ListNets(c *gin.Context) {
 	nets, cursor, err := ml.List(offset, limit, h.MLS)
 	if err != nil {
 		logger.Error("Failed to get list of nets", err)
-		c.JSON(http.StatusInternalServerError, types.APIError{Msg: "Error getting list of nets, see logs for more info"})
+		c.JSON(http.StatusInternalServerError, types.NewErrorRes("Error getting list of nets, see logs for more info"))
 		return
 	}
 	c.JSON(http.StatusOK, types.PagedRes{Last: cursor == 0, Next: cursor, Results: nets})
@@ -117,31 +115,31 @@ func (h *Handler) ListNets(c *gin.Context) {
 // @Summary Net train endpoint
 // @Description Used for training new or existing networks with the points from an existing series
 // @Accept json
-// @Success 200
-// @Failure 400 {object} types.APIError "When the request body is formatted incorrectly"
-// @Failure 404 {object} types.APIError "When the provided series ID isn't found"
-// @Failure 500 {object} types.APIError "When there is an error processing the request"
+// @Success 200 {object} types.SimpleRes
+// @Failure 400 {object} types.SimpleRes "When the request body is formatted incorrectly"
+// @Failure 404 {object} types.SimpleRes "When the provided series ID isn't found"
+// @Failure 500 {object} types.SimpleRes "When there is an error processing the request"
 // @Router /nets [post]
 func (h *Handler) Train(c *gin.Context) {
 	var tr types.TrainRequest
 	err := c.ShouldBind(&tr)
 	if err != nil {
 		logger.Debug("Failed to unmarshal message (" + err.Error() + ")")
-		c.JSON(http.StatusBadRequest, types.APIError{Msg: "Wrong format"})
+		c.JSON(http.StatusBadRequest, types.NewErrorRes("Wrong format"))
 		return
 	}
 	exists, err := h.PS.Exists(tr.SeriesID)
 	if err != nil {
-		logger.Error("Failed to check if series with id "+tr.SeriesID+" exists", err)
-		c.JSON(http.StatusInternalServerError, types.APIError{Msg: "Error processing training request, see logs for more info"})
+		logger.Error("Failed to check if series with ID "+tr.SeriesID+" exists", err)
+		c.JSON(http.StatusInternalServerError, types.NewErrorRes("Error processing training request, see logs for more info"))
 		return
 	}
 	if !exists {
-		c.JSON(http.StatusNotFound, types.APIError{Msg: "Series with id " + tr.SeriesID + " could not be found"})
+		c.JSON(http.StatusNotFound, types.NewErrorRes("Series with ID "+tr.SeriesID+" could not be found"))
 		return
 	}
 	sort.Strings(tr.Inputs)
 	sort.Strings(tr.Outputs)
 	h.TServ <- tr
-	c.JSON(http.StatusAccepted, "")
+	c.JSON(http.StatusAccepted, types.NewOkRes("Training request for series "+tr.SeriesID+" created successfully"))
 }