provider, packet, node, net, sdk/nodejs, cmd/flatend, cmd/microservic…

…e: allow for large amounts of data to be sent and received between microservices by implementing streams and chunked transfer encoding

cmd/flatend/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/julienschmidt/httprouter"
 	"github.com/lithdew/flatend"
 	"github.com/spf13/pflag"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -16,18 +17,10 @@ import (
 	"os/signal"
 	"strings"
 	"time"
-	"unicode/utf8"
 )
 
 var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
-type Request struct {
-	Header http.Header         `json:"header"`
-	Query  url.Values          `json:"query"`
-	Params map[string]string   `json:"params"`
-	Body   jsoniter.RawMessage `json:"body"`
-}
-
 type Config struct {
 	HTTP []ConfigHTTP
 }
@@ -206,48 +199,30 @@ func main() {
 			}
 
 			handler := func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
-				req := Request{
-					Header: r.Header,
-					Query:  r.URL.Query(),
+				headers := make(map[string]string)
+				for key := range r.Header {
+					headers[strings.ToLower(key)] = r.Header.Get(key)
 				}
 
-				req.Params = make(map[string]string, len(params))
-				for _, param := range params {
-					req.Params[param.Key] = param.Value
+				for key := range r.URL.Query() {
+					headers["query."+strings.ToLower(key)] = r.URL.Query().Get(key)
 				}
 
-				body, err := ioutil.ReadAll(r.Body)
-				if err != nil {
-					return
-				}
-				if len(body) == 0 {
-					body = nil
-				}
-
-				if utf8.Valid(body) {
-					req.Body, err = json.Marshal(string(body))
-					if err != nil {
-						return
-					}
-				} else {
-					req.Body, err = json.Marshal(body)
-					if err != nil {
-						return
-					}
+				for _, param := range params {
+					headers["params."+strings.ToLower(param.Key)] = param.Value
 				}
 
-				buf, err := json.Marshal(req)
+				stream, err := node.Push(services, headers, r.Body)
 				if err != nil {
+					w.Write([]byte(err.Error()))
 					return
 				}
 
-				res, err := node.Request(services, buf)
-				if err != nil {
-					w.Write([]byte(err.Error()))
-					return
+				for name, val := range stream.Header.Headers {
+					w.Header().Set(name, val)
 				}
 
-				w.Write(res)
+				_, _ = io.Copy(w, stream.Reader)
 			}
 
 			router.Handle(fields[0], fields[1], handler)

cmd/microservice/main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"encoding/json"
 	"github.com/lithdew/flatend"
+	"io"
 	"os"
 	"os/signal"
 	"strconv"
@@ -16,19 +18,28 @@ func check(err error) {
 
 var counter uint64 = 0
 
-func handleAllTodos(_ *flatend.Context) []byte {
-	return strconv.AppendUint(nil, atomic.AddUint64(&counter, 1), 10)
+func handleAllTodos(ctx *flatend.Context) {
+	ctx.Write(strconv.AppendUint(nil, atomic.AddUint64(&counter, 1), 10))
 }
 
-func handleGetTodos(ctx *flatend.Context) []byte {
-	return ctx.Body()
+func handleGetTodos(ctx *flatend.Context) {
+	buf, err := json.Marshal(ctx.Headers)
+	if err != nil {
+		return
+	}
+	ctx.Write(buf)
+}
+
+func pipe(ctx *flatend.Context) {
+	io.Copy(ctx, ctx.Body)
 }
 
 func main() {
 	node := &flatend.Node{
 		Services: map[string]flatend.Handler{
 			"all_todos": handleAllTodos,
 			"get_todos": handleGetTodos,
+			"pipe":      pipe,
 		},
 	}
 	check(node.Start("0.0.0.0:9000"))

config.toml

@@ -22,5 +22,5 @@ path = "GET /todos/:id"
 service = "get_todos"
 
 [[http.routes]]
-path = "POST /todos"
-service = "create_todo"
+path = "POST /pipe"
+service = "pipe"

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/lithdew/kademlia v0.0.0-20200613105650-31ee52d03942
 	github.com/lithdew/monte v0.0.0-20200613174427-105bad77ff52
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.6.0 // indirect
+	github.com/stretchr/testify v1.6.0
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
 	golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect
 )