Merge pull request #13 from RTradeLtd/delegator

TEM-155: network routing scheme

.gitignore

@@ -1,3 +1,6 @@
+# editors
+.vscode
+
 # dependencies
 vendor
 
@@ -11,3 +14,4 @@ vendor
 # test
 tmp
 test
+temp.txt

Makefile

@@ -14,10 +14,6 @@ build:
 install: deps
 	go install -ldflags "-X main.Version=$(VERSION)" ./cmd/nexus
 
-.PHONY: config
-config: build
-	./nexus -config ./config.example.json init
-
 # Install dependencies
 .PHONY: deps
 deps:
@@ -56,7 +52,47 @@ gen:
 	fileb0x b0x.yml
 	counterfeiter -o ./ipfs/mock/ipfs.mock.go \
 		./ipfs/ipfs.go NodeClient
+	counterfeiter -o ./temporal/mock/networks.mock.go \
+		./temporal/database.go PrivateNetworks
 
 .PHONY: release
 release:
 	bash .scripts/release.sh
+
+#####################
+# DEVELOPMENT UTILS #
+#####################
+
+NETWORK=test_network
+TESTFLAGS=-dev -config ./config.dev.json
+
+.PHONY: example-config
+example-config: build
+	./nexus -config ./config.example.json init
+
+.PHONY: dev-config
+dev-config: build
+	./nexus $(TESTFLAGS) init
+
+.PHONY: config
+config: example-config dev-config
+
+.PHONY: daemon
+daemon: build
+	./nexus $(TESTFLAGS) daemon
+
+.PHONY: new-network
+new-network: build
+	./nexus $(TESTFLAGS) dev network $(NETWORK)
+
+.PHONY: start-network
+start-network: build
+	./nexus $(TESTFLAGS) ctl --pretty StartNetwork Network=$(NETWORK)
+
+.PHONY: stat-network
+stat-network:
+	./nexus $(TESTFLAGS) ctl --pretty NetworkStats Network=$(NETWORK)
+
+.PHONY: diag-network
+diag-network:
+	./nexus $(TESTFLAGS) ctl NetworkDiagnostics Network=$(NETWORK)

README.md

@@ -1,13 +1,18 @@
-# 🦑 Nexus: IPFS Network Node Orchestrator
+# 🦑 Nexus
 
-Nexus is the IPFS private network node orchestration and registry service for
-[Temporal](https://github.com/RTradeLtd/Temporal), an easy-to-use interface into
-distributed and decentralized storage technologies.
+> IPFS Network Node Orchestrator
+
+Nexus is the [IPFS](https://github.com/ipfs/go-ipfs) private network node
+orchestration and registry service for [Temporal](https://github.com/RTradeLtd/Temporal),
+an easy-to-use interface into distributed and decentralized storage technologies.
+Nexus handles on-demand deployment, resource management, metadata persistence,
+and fine-grained access control for IPFS nodes running within Docker containers.
 
 [![GoDoc](https://godoc.org/github.com/RTradeLtd/Nexus?status.svg)](https://godoc.org/github.com/RTradeLtd/Nexus)
 [![Build Status](https://travis-ci.com/RTradeLtd/Nexus.svg?branch=master)](https://travis-ci.com/RTradeLtd/Nexus)
 [![codecov](https://codecov.io/gh/RTradeLtd/Nexus/branch/master/graph/badge.svg)](https://codecov.io/gh/RTradeLtd/Nexus)
 [![Go Report Card](https://goreportcard.com/badge/github.com/RTradeLtd/Nexus)](https://goreportcard.com/report/github.com/RTradeLtd/Nexus)
+[![Latest Release](https://img.shields.io/github/release/RTradeLtd/Nexus.svg?colorB=red)](https://github.com/RTradeLtd/Nexus/releases)
 
 ## Installation and Usage
 
@@ -16,7 +21,8 @@ $> go get -u github.com/RTradeLtd/Nexus/cmd/nexus
 ```
 
 Releases are also be available from the
-[Releases](https://github.com/RTradeLtd/Nexus/releases) page.
+[Releases](https://github.com/RTradeLtd/Nexus/releases) page. To start up the
+Nexus daemon using the default configuration, run:
 
 ```bash
 $> nexus init
@@ -54,6 +60,23 @@ $> make test
 
 You can remove leftover assets using `make clean`.
 
+### Running Locally
+
+A few make commands make it easy to simulate a full orchestrator environment on your machine:
+
+```bash
+$> make dev-config # make sure dev configuration is up to date
+$> make testenv    # initialize test environment
+$> make daemon     # start up daemon with dev configuration
+```
+
+Then, you can set up and start a network node:
+
+```bash
+$> make new-network   # create network entry in database
+$> make start-network # spin up network node
+```
+
 ### ctl
 
 An experimental, lightweight controller for the gRPC API is available via the

client/client.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/RTradeLtd/grpc/dialer"
-	ipfs_orchestrator "github.com/RTradeLtd/grpc/ipfs-orchestrator"
+	"github.com/RTradeLtd/grpc/nexus"
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -15,7 +15,7 @@ import (
 // IPFSOrchestratorClient is a lighweight container for the orchestrator's
 // gRPC API client
 type IPFSOrchestratorClient struct {
-	ipfs_orchestrator.ServiceClient
+	nexus.ServiceClient
 	grpc *grpc.ClientConn
 }
 
@@ -48,7 +48,7 @@ func New(opts config.API, devMode bool) (*IPFSOrchestratorClient, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to core service: %s", err.Error())
 	}
-	c.ServiceClient = ipfs_orchestrator.NewServiceClient(c.grpc)
+	c.ServiceClient = nexus.NewServiceClient(c.grpc)
 	return c, nil
 }
 

cmd/nexus/daemon.go

@@ -5,9 +5,15 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
+
+	tcfg "github.com/RTradeLtd/config"
+	"github.com/RTradeLtd/database"
+	"github.com/RTradeLtd/database/models"
 
 	"github.com/RTradeLtd/Nexus/config"
 	"github.com/RTradeLtd/Nexus/daemon"
+	"github.com/RTradeLtd/Nexus/delegator"
 	"github.com/RTradeLtd/Nexus/ipfs"
 	"github.com/RTradeLtd/Nexus/log"
 	"github.com/RTradeLtd/Nexus/orchestrator"
@@ -41,30 +47,76 @@ func runDaemon(configPath string, devMode bool, args []string) {
 		fatal(err.Error())
 	}
 
+	// set up database connection
+	l.Infow("intializing database connection",
+		"db.host", cfg.Database.URL,
+		"db.port", cfg.Database.Port,
+		"db.name", cfg.Database.Name,
+		"db.with_ssl", !devMode,
+		"db.with_migrations", devMode)
+	dbm, err := database.Initialize(&tcfg.TemporalConfig{
+		Database: cfg.Database,
+	}, database.Options{
+		SSLModeDisable: devMode,
+		RunMigrations:  devMode,
+	})
+	if err != nil {
+		l.Errorw("failed to connect to database", "error", err)
+		fatalf("unable to connect to database: %s", err.Error())
+	}
+	l.Info("successfully connected to database")
+	defer func() {
+		if err := dbm.DB.Close(); err != nil {
+			l.Warnw("error occurred closing database connection",
+				"error", err)
+		}
+	}()
+
 	// initialize orchestrator
 	println("initializing orchestrator")
-	o, err := orchestrator.New(l, cfg.Address, c, cfg.IPFS.Ports, cfg.Database, devMode)
+	o, err := orchestrator.New(l, cfg.Address, cfg.IPFS.Ports, devMode,
+		c, models.NewHostedIPFSNetworkManager(dbm.DB))
 	if err != nil {
 		fatal(err.Error())
 	}
 
 	// initialize daemon
 	println("initializing daemon")
-	d := daemon.New(l, o)
+	dm := daemon.New(l, o)
+
+	// initialize delegator
+	println("initializing delegator")
+	dl := delegator.New(l, Version, 1*time.Minute, []byte(cfg.Delegator.JWTKey),
+		o.Registry, models.NewHostedIPFSNetworkManager(dbm.DB))
 
-	// handle graceful shutdown
+	// catch interrupts
 	ctx, cancel := context.WithCancel(context.Background())
-	signals := make(chan os.Signal)
+	var signals = make(chan os.Signal)
 	signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
 	go func() {
 		<-signals
 		cancel()
 	}()
 
-	// serve endpoints
-	println("spinning up server")
-	if err := d.Run(ctx, cfg.API); err != nil {
-		println(err.Error())
-	}
-	println("server shut down")
+	// serve gRPC endpoints
+	println("spinning up gRPC server...")
+	go func() {
+		if err := dm.Run(ctx, cfg.API); err != nil {
+			println(err.Error())
+		}
+		cancel()
+	}()
+
+	// serve delegator
+	println("spinning up delegator...")
+	go func() {
+		if err := dl.Run(ctx, cfg.Delegator); err != nil {
+			println(err.Error())
+		}
+		cancel()
+	}()
+
+	// block
+	<-ctx.Done()
+	println("orchestrator shut down")
 }

cmd/nexus/dev.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/RTradeLtd/Nexus/config"
+	tcfg "github.com/RTradeLtd/config"
+	"github.com/RTradeLtd/database"
+	"github.com/RTradeLtd/database/models"
+)
+
+func initTestNetwork(configPath, networkName string) {
+	// load configuration
+	cfg, err := config.LoadConfig(configPath)
+	if err != nil {
+		fatal(err.Error())
+	}
+
+	println("setting up database entry for a test network")
+
+	dbm, err := database.Initialize(&tcfg.TemporalConfig{
+		Database: cfg.Database,
+	}, database.Options{
+		SSLModeDisable: true,
+		RunMigrations:  true,
+	})
+	if err != nil {
+		fatal(err.Error())
+	}
+
+	var nm = models.NewHostedIPFSNetworkManager(dbm.DB)
+	if _, err := nm.CreateHostedPrivateNetwork(networkName, "", nil, models.NetworkAccessOptions{
+		Users:         []string{"testuser"},
+		PublicGateway: true,
+	}); err != nil {
+		fatal(err.Error())
+	}
+}

cmd/nexus/main.go

@@ -27,9 +27,11 @@ USAGE:
 COMMANDS:
 
   init        initialize configuration
-  daemon      spin up the Nexus daemon and related processes
-  ctl         [EXPERIMENTAL] interact with daemon via a low-level client
-  version     display program version
+	daemon      spin up the Nexus daemon and related processes
+	version     display program version
+
+	dev         [DEV] utilities for development purposes
+	ctl         [EXPERIMENTAL] interact with daemon via a low-level client
 
 OPTIONS:
 
@@ -54,33 +56,60 @@ func main() {
 
 	if len(args) >= 1 {
 		switch args[0] {
+		// version info
 		case "version":
 			println("Nexus " + Version)
 		case "init":
-			config.GenerateConfig(*configPath)
+			config.GenerateConfig(*configPath, *devMode)
 			println("orchestrator configuration generated at " + *configPath)
 			return
+		// run daemon
 		case "daemon":
 			runDaemon(*configPath, *devMode, args[1:])
 			return
+		// run ctl
 		case "ctl":
 			if len(args) > 1 && (args[1] == "-pretty" || args[1] == "--pretty") {
 				runCTL(*configPath, *devMode, true, args[2:])
 			} else {
 				runCTL(*configPath, *devMode, false, args[1:])
 			}
 			return
+		// dev utilities
+		case "dev":
+			if *devMode != true {
+				fatal("do not use the dev utilities outside of dev mode!")
+			}
+			if len(args) > 1 {
+				switch args[1] {
+				case "network":
+					if len(args) < 2 {
+						fatal("additional argument required")
+					}
+					initTestNetwork(*configPath, args[2])
+				default:
+					fatal(fmt.Sprintf("unknown command '%s' - user the --help' flag for documentation",
+						strings.Join(args[0:], " ")))
+				}
+			}
+		// default error
 		default:
 			fatal(fmt.Sprintf("unknown command '%s' - user the --help' flag for documentation",
 				strings.Join(args[0:], " ")))
 			return
 		}
 	} else {
-		fatal("no arguments provided - use the '--help' flag for documentation")
+		fatal("insufficient arguments provided - use the '--help' flag for documentation")
 	}
 }
 
 func fatal(msg ...interface{}) {
 	fmt.Println(msg...)
 	os.Exit(1)
 }
+
+func fatalf(format string, msg ...interface{}) {
+	fmt.Printf(format, msg...)
+	println()
+	os.Exit(1)
+}