Add Postgres, GraphQL, and Dataloaden

Dockerfile

@@ -17,16 +17,21 @@ WORKDIR /bitclout/src/backend
 RUN go mod download
 
 # include backend src
-COPY backend/config  config
-COPY backend/cmd     cmd
-COPY backend/miner   miner
-COPY backend/routes  routes
-COPY backend/main.go .
+COPY backend/config     config
+COPY backend/cmd        cmd
+COPY backend/dataloader dataloader
+COPY backend/globaldb   globaldb
+COPY backend/graph      graph
+COPY backend/migrate    migrate
+COPY backend/miner      miner
+COPY backend/routes     routes
+COPY backend/main.go    .
 
 # include core src
 COPY core/clouthash ../core/clouthash
 COPY core/cmd       ../core/cmd
 COPY core/lib       ../core/lib
+COPY core/migrate   ../core/migrate
 
 # build backend
 RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/backend main.go

cmd/node.go

@@ -2,18 +2,23 @@ package cmd
 
 import (
 	"github.com/bitclout/backend/config"
+	"github.com/bitclout/backend/globaldb"
+	"github.com/bitclout/backend/migrate"
 	"github.com/bitclout/backend/routes"
 	coreCmd "github.com/bitclout/core/cmd"
 	"github.com/bitclout/core/lib"
 	"github.com/dgraph-io/badger/v3"
+	"github.com/go-pg/pg/v10"
 	"github.com/golang/glog"
 	"github.com/kevinburke/twilio-go"
+	migrations "github.com/robinjoseph08/go-pg-migrations/v3"
 	"path/filepath"
 )
 
 type Node struct {
 	APIServer   *routes.APIServer
 	GlobalState *badger.DB
+	GlobalDB    *globaldb.GlobalDB
 	Config      *config.Config
 
 	CoreNode *coreCmd.Node
@@ -45,6 +50,30 @@ func (node *Node) Start() {
 		}
 	}
 
+	// Default to core node postgres once fully transitioned
+	//if node.Config.GlobalStatePostgresURI == "" {
+	//	node.Config.GlobalStatePostgresURI = node.CoreNode.Config.PostgresURI
+	//}
+
+	// Connect to global DB postgres
+	if node.Config.GlobalStatePostgresURI != "" {
+		options, err := pg.ParseURL(node.Config.GlobalStatePostgresURI)
+		if err != nil {
+			panic(err)
+		}
+
+		db := pg.Connect(options)
+
+		// Make sure we're migrated
+		migrate.LoadMigrations()
+		err = migrations.Run(db, "migrate", []string{"", "migrate"})
+		if err != nil {
+			panic(err)
+		}
+
+		node.GlobalDB = globaldb.NewGlobalDB(db)
+	}
+
 	var twilioClient *twilio.Client
 	if node.Config.TwilioAccountSID != "" {
 		twilioClient = twilio.NewClient(node.Config.TwilioAccountSID, node.Config.TwilioAuthToken, nil)
@@ -56,6 +85,8 @@ func (node *Node) Start() {
 		node.CoreNode.Server.GetBlockchain(),
 		node.CoreNode.Server.GetBlockProducer(),
 		node.CoreNode.TXIndex,
+		node.CoreNode.Postgres,
+		node.GlobalDB,
 		node.CoreNode.Params,
 		node.Config,
 		node.CoreNode.Config.MinFeerate,

cmd/run.go

@@ -85,6 +85,10 @@ func init() {
 	runCmd.PersistentFlags().String("global-state-remote-secret", "",
 		"When a remote node is being used to set/fetch global state, a secret "+
 			"is also required to restrict access.")
+	runCmd.PersistentFlags().String("global-state-postgres-uri", "",
+		"The postgres database to use for global state. Defaults to postgres-uri. Global state"+
+			"is used to manage things like user data, e.g. emails, that should not be duplicated across"+
+			"multiple nodes and are not stored in consensus.")
 
 	// Web Security
 	runCmd.PersistentFlags().StringSlice("access-control-allow-origins", []string{"*"},

config/config.go

@@ -25,6 +25,7 @@ type Config struct {
 	// Global State
 	GlobalStateRemoteNode   string
 	GlobalStateRemoteSecret string
+	GlobalStatePostgresURI  string
 
 	// Web Security
 	AccessControlAllowOrigins []string
@@ -100,6 +101,7 @@ func LoadConfig(coreConfig *coreCmd.Config) *Config {
 	// Global State
 	config.GlobalStateRemoteNode = viper.GetString("global-state-remote-node")
 	config.GlobalStateRemoteSecret = viper.GetString("global-state-remote-secret")
+	config.GlobalStatePostgresURI = viper.GetString("global-state-postgres-uri")
 
 	// Web Security
 	config.AccessControlAllowOrigins = viper.GetStringSlice("access-control-allow-origins")

globaldb/block.go

@@ -0,0 +1,18 @@
+package globaldb
+
+import "github.com/bitclout/core/lib"
+
+type Block struct {
+	PublicKey        *lib.PublicKey `pg:",pk"`
+	BlockedPublicKey *lib.PublicKey `pg:",pk"`
+	BlockedAt        uint64
+}
+
+func (global *GlobalDB) GetBlocks(publicKey *lib.PublicKey) []*Block {
+	var blocks []*Block
+	err := global.db.Model(&blocks).Where("public_key = ?", publicKey).Select()
+	if err != nil {
+		return nil
+	}
+	return blocks
+}

globaldb/core.go

@@ -0,0 +1,22 @@
+package globaldb
+
+import "github.com/go-pg/pg/v10"
+
+type GlobalDB struct {
+	db *pg.DB
+}
+
+func NewGlobalDB(db *pg.DB) *GlobalDB {
+	return &GlobalDB{
+		db: db,
+	}
+}
+
+type AuditLog struct {
+}
+
+type GlobalFeedPost struct {
+}
+
+type NodeParam struct {
+}