Initial implementation (#7)

* alpha 0.0.1

* some refactor

* added config json parsing check

* changed package name

* added documentation

* added logrus for logging

* better err handling, added couple of tests

* moved intervals loop from go subroutine to main

* small fixes

* changed config struct to mixedCaps

* changed intervalcall functionality

* changed documentation and config location

* changed config file

* added dockerfile

* changed flag to bool, select to range, added err check on unmarshall, changed pow to const, made checkInterval function more clear

* divided main function

* removed grpc test

* omitted debug flag bool

* go mod tidy

* read config.json volume documentation

* removed redundant []Market{Market}

* changed docker documentation

* created config.Markets types.go file

* changed readme

* changed readme

* removed gRPC response log

* added assert in test

Co-authored-by: FrancisMars <[email protected]>

Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.15.5-buster AS builder
+
+WORKDIR /tdex-feeder
+
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
+
+RUN go build -o feederd-linux cmd/feederd/main.go
+
+WORKDIR /build
+
+RUN cp /tdex-feeder/feederd-linux .
+
+FROM debian:buster
+
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
+
+COPY --from=builder /build/ /
+
+CMD ["/feederd-linux","-debug","-conf=./data/config.json"]

Makefile

@@ -41,11 +41,11 @@ help:
 
 ## run-linux: Run locally with default configuration
 run-linux: clean build-linux
-	./build/tdexd-linux-amd64
+	./build/feederd-linux-amd64
 
 ## run-mac: Run locally with default configuration
 run-mac: clean build-mac
-	./build/tdexd-darwin-amd64
+	./build/feederd-darwin-amd64
 
 ## vet: code analysis
 vet:

README.md

@@ -1,18 +1,96 @@
 # tdex-feeder
+
 Feeder allows to connect an external price feed to the TDex Daemon to determine the current market price
 
-## ⬇️ Install
+## Overview
+
+tdex-feeder connects to exchanges and retrieves market prices in order to consume the gRPC 
+interface exposed from tdex-deamon `UpdateMarketPrice`.
+
+## ⬇️ Run  Standalone
+
+### Install
+
+1. [Download the latest release for MacOS or Linux](https://github.com/tdex-network/tdex-feeder/releases)
+
+2. Move the feeder into a folder in your PATH (eg. `/usr/local/bin`) and rename the feeder as `feederd`
 
-TBD
+3. Give executable permissions. (eg. `chmod a+x /usr/local/bin/feederd`)
 
-## 📄 Usage
+4. Create [config.json](#config-file) file.
 
-In-depth documentation for using the tdex-feeder is available at [docs.tdex.network](https://docs.tdex.network/tdex-feeder.html)
+### Run
+```sh
+# Run with default config and default flags.
+$ feederd
 
+# Run with debug mode on.
+$ feederd -debug
 
+# Run with debug mode and different config path.
+$ feederd -debug -conf=./config.json
+```
 
 ## 🖥 Local Development
 
 Below is a list of commands you will probably find useful.
 
-TBD
+### Build and Run with docker
+
+Build and use `feederd` with docker.
+
+#### Build feederd docker image
+
+At the root of the repository
+```
+docker build -t tdex-feederd .
+```
+
+#### Run the daemon
+
+Create a [config.json](#config-file) file 
+and run the following command in the same folder:
+```
+docker run -it -d --net=host -v $PWD/config.json:/data/config.json tdex-feederd
+```
+`--net=host` in case you're running tdex-deamon locally
+
+### Build it yourself
+
+Builds feeder as static binary and runs the project with default configuration.
+
+#### Linux
+
+`make build-linux`
+
+#### Mac
+
+`make build-mac`
+
+#### Run Linux
+
+`make run-linux`
+
+##### Flags
+
+```
+-conf: Configuration File Path. Default: "./config.json"
+-debug: Log Debug Informations Default: false
+```
+
+##### Config file
+
+Rename the file `./config.example.json` into `./config.json` 
+and adapt if for your specific purpose. The default example
+connects to kraken socket and to a local instance of tdex-deamon.
+
+```
+daemon_endpoint: String with the address and port of gRPC host. Required.
+daemon_macaroon: String with the daemon_macaroon necessary for authentication.
+kraken_ws_endpoint: String with the address and port of kraken socket. Required.
+markets: Json List with necessary markets informations. Required.
+base_asset: String of the Hash of the base asset for gRPC request. Required.
+quote_asset: String of the Hash of the quote asset for gRPC request. Required.
+kraken_ticker: String with the ticker we want kraken to provide informations on. Required.
+interval: Int with the time in secods between gRPC requests. Required.
+```

cmd/feederd/main.go

@@ -1,5 +1,114 @@
+// Copyright (c) 2020 The VulpemVentures developers
+
+// Feeder allows to connect an external price feed to the TDex Daemon to determine the current market price.
 package main
 
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+
+	"github.com/gorilla/websocket"
+	"github.com/tdex-network/tdex-feeder/config"
+	"github.com/tdex-network/tdex-feeder/pkg/conn"
+	"github.com/tdex-network/tdex-feeder/pkg/marketinfo"
+
+	pboperator "github.com/tdex-network/tdex-protobuf/generated/go/operator"
+)
+
+const (
+	defaultConfigPath = "./config.json"
+)
+
 func main() {
+	interrupt, cSocket, marketsInfos, conngRPC := setup()
+	infiniteLoops(interrupt, cSocket, marketsInfos, conngRPC)
+}
+
+func setup() (chan os.Signal, *websocket.Conn, []marketinfo.MarketInfo, *grpc.ClientConn) {
+	conf := checkFlags()
+
+	// Interrupt Notification.
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	// Dials the connection the the Socket.
+	cSocket, err := conn.ConnectToSocket(conf.KrakenWsEndpoint)
+	if err != nil {
+		log.Fatal("Socket Connection Error: ", err)
+	}
+	marketsInfos := loadMarkets(conf, cSocket)
+
+	// Set up the connection to the gRPC server.
+	conngRPC, err := conn.ConnectTogRPC(conf.DaemonEndpoint)
+	if err != nil {
+		log.Fatal("gRPC Connection Error: ", err)
+	}
+	return interrupt, cSocket, marketsInfos, conngRPC
+}
+
+// Checks for command line flags for Config Path and Debug mode.
+// Loads flags as required.
+func checkFlags() config.Config {
+	confFlag := flag.String("conf", defaultConfigPath, "Configuration File Path")
+	debugFlag := flag.Bool("debug", false, "Log Debug Informations")
+	flag.Parse()
+	if *debugFlag == true {
+		log.SetLevel(log.DebugLevel)
+	}
+	// Loads Config File.
+	conf, err := config.LoadConfig(*confFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return conf
+}
+
+// Loads Config Markets infos into Data Structure and Subscribes to
+// Messages from this Markets.
+func loadMarkets(conf config.Config, cSocket *websocket.Conn) []marketinfo.MarketInfo {
+	numberOfMarkets := len(conf.Markets)
+	marketsInfos := make([]marketinfo.MarketInfo, numberOfMarkets)
+	for i, marketConfig := range conf.Markets {
+		marketsInfos[i] = marketinfo.InitialMarketInfo(marketConfig)
+		m := conn.CreateSubscribeToMarketMessage(marketConfig.KrakenTicker)
+		err := conn.SendRequestMessage(cSocket, m)
+		if err != nil {
+			log.Fatal("Couldn't send request message: ", err)
+		}
+	}
+	return marketsInfos
+}
+
+func infiniteLoops(interrupt chan os.Signal, cSocket *websocket.Conn, marketsInfos []marketinfo.MarketInfo, conngRPC *grpc.ClientConn) {
+	defer cSocket.Close()
+	defer conngRPC.Close()
+	clientgRPC := pboperator.NewOperatorClient(conngRPC)
+	done := make(chan string)
+	// Handles Messages from subscriptions. Will periodically call the
+	// gRPC UpdateMarketPrice with the price info from the messages.
+	go conn.HandleMessages(done, cSocket, marketsInfos, clientgRPC)
+	checkInterrupt(interrupt, cSocket, done)
+}
 
+// Loop to keep cycle alive. Waits Interrupt to close the connection.
+func checkInterrupt(interrupt chan os.Signal, cSocket *websocket.Conn, done chan string) {
+	for {
+		for range interrupt {
+			log.Println("Shutting down Feeder")
+			err := cSocket.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+			if err != nil {
+				log.Fatal("write close:", err)
+			}
+			select {
+			case <-done:
+			case <-time.After(time.Second):
+			}
+			return
+		}
+	}
 }

config.example.json

@@ -0,0 +1,19 @@
+{
+  "daemon_endpoint": "localhost:9000",
+  "daemon_macaroon": "string",
+  "kraken_ws_endpoint": "ws.kraken.com",
+  "markets": [
+    {
+      "base_asset": "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225",
+      "quote_asset":"d73f5cd0954c1bf325f85d7a7ff43a6eb3ea3b516fd57064b85306d43bc1c9ff",
+      "kraken_ticker": "XBT/USD",
+      "interval": 10
+    },
+    {
+      "base_asset": "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225",
+      "quote_asset":"d090c403610fe8a9e31967355929833bc8a8fe08429e630162d1ecbf29fdf28b",
+      "kraken_ticker": "XBT/EUR",
+      "interval": 15
+    } 
+  ]
+}

config/config.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultDaemonEndpoint   = "localhost:9000"
+	defaultKrakenWsEndpoint = "ws.kraken.com"
+	defaultBaseAsset        = "5ac9f65c0efcc4775e0baec4ec03abdde22473cd3cf33c0419ca290e0751b225"
+	defaultQuoteAsset       = "d73f5cd0954c1bf325f85d7a7ff43a6eb3ea3b516fd57064b85306d43bc1c9ff"
+	defaultKrakenTicker     = "XBT/USD"
+	defaultInterval         = 10
+)
+
+type Config struct {
+	DaemonEndpoint   string   `json:"daemon_endpoint,required"`
+	DaemonMacaroon   string   `json:"daemon_macaroon"`
+	KrakenWsEndpoint string   `json:"kraken_ws_endpoint,required"`
+	Markets          []Market `json:"markets,required"`
+}
+
+// DefaultConfig returns the datastructure needed
+// for a default connection.
+func defaultConfig() Config {
+	return Config{
+		DaemonEndpoint:   defaultDaemonEndpoint,
+		KrakenWsEndpoint: defaultKrakenWsEndpoint,
+		Markets: []Market{
+			{
+				BaseAsset:    defaultBaseAsset,
+				QuoteAsset:   defaultQuoteAsset,
+				KrakenTicker: defaultKrakenTicker,
+				Interval:     defaultInterval,
+			},
+		},
+	}
+}
+
+// LoadConfigFromFile reads a file with the intended running behaviour
+// and returns a Config struct with the respective configurations.
+func loadConfigFromFile(filePath string) (Config, error) {
+	jsonFile, err := os.Open(filePath)
+	if err != nil {
+		return Config{}, err
+	}
+	defer jsonFile.Close()
+
+	var config Config
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return Config{}, err
+	}
+	err = json.Unmarshal(byteValue, &config)
+	if err != nil {
+		return Config{}, err
+	}
+	err = checkConfigParsing(config)
+	if err != nil {
+		return Config{}, err
+	}
+
+	return config, nil
+}
+
+// checkConfigParsing checks if all the required fields
+// were correctly loaded into the Config struct.
+func checkConfigParsing(config Config) error {
+	fields := reflect.ValueOf(config)
+	for i := 0; i < fields.NumField(); i++ {
+		tags := fields.Type().Field(i).Tag
+		if strings.Contains(string(tags), "required") && fields.Field(i).IsZero() {
+			return errors.New("Config required field is missing: " + string(tags))
+		}
+	}
+	for _, market := range config.Markets {
+		fields := reflect.ValueOf(market)
+		for i := 0; i < fields.NumField(); i++ {
+			tags := fields.Type().Field(i).Tag
+			if strings.Contains(string(tags), "required") && fields.Field(i).IsZero() {
+				return errors.New("Config required field is missing: " + string(tags))
+			}
+		}
+	}
+	return nil
+}
+
+// LoadConfig handles the default behaviour for loading
+// config.json files. In case the file is not found,
+// it loads the default config.
+func LoadConfig(filePath string) (Config, error) {
+	_, err := os.Stat(filePath)
+	if os.IsNotExist(err) {
+		log.Printf("File not found: %s. Loading default config.\n", filePath)
+		return defaultConfig(), nil
+	}
+	return loadConfigFromFile(filePath)
+}

config/types.go

@@ -0,0 +1,8 @@
+package config
+
+type Market struct {
+	BaseAsset    string `json:"base_asset,required"`
+	QuoteAsset   string `json:"quote_asset,required"`
+	KrakenTicker string `json:"kraken_ticker,required"`
+	Interval     int    `json:"interval,required"`
+}