First implementation of services

.gitignore

@@ -0,0 +1 @@
+/bin

Makefile

@@ -8,3 +8,14 @@ fmt:
 	@echo "Gofmt..."
 	@gofmt -w -l .
 
+test:
+	@echo "Testing..."
+	@go test -v ./...
+
+clean:
+	@echo "Cleaning..."
+	@rm -rf ./bin
+
+build:
+	@echo "Building..."
+	@go build -v -o bin/neutrinoelements-cli ./cmd

README.md

@@ -4,17 +4,31 @@ Neutrino + Elements
 
 ## Description
 
-neutrino-elements uses Compact Block Filter (BIP158) to implement a light client for [elements](https://elementsproject.org/)-based networks.
-
-![architecture diagram](diagram.png)
+neutrino-elements uses Compact Block Filter (BIP0158) to implement a light client for [elements](https://elementsproject.org/)-based networks.
 
 Two services are provided, they can work independantly:
-- `PeersPool` maintains a set of peers in order to keep an up-to-date state of the blockchain. It uses the `Peer` interface to fetch two types of data: block headers and compact filters. Then these data are stored using [repositories](https://deviq.com/design-patterns/repository-pattern).
-- `UtxoScanner` uses filters and headers repositories to handle `ScanRequest` which aims to know if an outpoint (identified by its script) is spent or not. 
+- `NodeService` is a full node maintaining an up-to-date state of the block headers + compact filters. The NodeService writes down headers and filters in repositories.
+- `ScannerService` uses filters and headers repositories to handle `ScanRequest` which aims to know if an outpoint (identified by its script) is spent or not. 
 
 ## Getting Started
 
-TODO
+### Build
+
+```
+make build
+```
+
+### Unit tests
+
+```
+make test
+```
+
+### Format
+
+```
+make fmt
+```
 
 ## License
 
@@ -24,3 +38,4 @@ MIT - see the LICENSE.md file for details
 
 * [Neutrino - Light bitcoin client](https://github.com/lightninglabs/neutrino)
 * [Compact Block Filters for Light Clients - BIP158](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki)
+* [tinybit](https://github.com/Jeiwan/tinybit)

cmd/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	"github.com/vulpemventures/neutrino-elements/pkg/repository"
+	"github.com/vulpemventures/neutrino-elements/pkg/repository/inmemory"
+)
+
+type State struct {
+	filtersDB      repository.FilterRepository
+	blockHeadersDB repository.BlockHeaderRepository
+}
+
+func main() {
+	logrus.SetLevel(logrus.DebugLevel)
+
+	state := &State{
+		filtersDB:      inmemory.NewFilterInmemory(),
+		blockHeadersDB: inmemory.NewHeaderInmemory(),
+	}
+
+	app := cli.NewApp()
+	app.Name = "neutrino-elements"
+	app.Version = "0.0.1"
+	app.Usage = "elements node + utxos scanner"
+	app.Commands = []*cli.Command{
+		{
+			Name:   "start",
+			Usage:  "run neutrino-elements node",
+			Action: startAction(state),
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/start.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	"github.com/vulpemventures/neutrino-elements/pkg/blockservice"
+	"github.com/vulpemventures/neutrino-elements/pkg/node"
+	"github.com/vulpemventures/neutrino-elements/pkg/protocol"
+	"github.com/vulpemventures/neutrino-elements/pkg/scanner"
+)
+
+func startAction(state *State) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		// Create a new peer node.
+		node, err := node.New(node.NodeConfig{
+			Network:        "nigiri",
+			UserAgent:      "neutrino-elements:0.0.1",
+			FiltersDB:      state.filtersDB,
+			BlockHeadersDB: state.blockHeadersDB,
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		// err = node.Run("liquid-testnet.blockstream.com:18892") // testnet
+		err = node.Start("localhost:18886") // regtest
+		if err != nil {
+			panic(err)
+		}
+
+		genesisBlockHash := protocol.GetCheckpoints(protocol.MagicNigiri)[0]
+		h, err := chainhash.NewHashFromStr(genesisBlockHash)
+		if err != nil {
+			panic(err)
+		}
+
+		blockSvc := blockservice.NewEsploraBlockService("http://localhost:3001")
+		scanSvc := scanner.New(state.filtersDB, state.blockHeadersDB, blockSvc, h)
+		reportCh, err := scanSvc.Start()
+		if err != nil {
+			panic(err)
+		}
+
+		go func() {
+			for report := range reportCh {
+				logrus.Infof("SCAN RESOLVE: %+v", report.Transaction.TxHash())
+			}
+		}()
+
+		// we'll watch if this address receives fund
+		watchItem, err := scanner.NewScriptWatchItemFromAddress("el1qq2enu72g3m306antkz6az3r8qklsjt62p2vt3mlfyaxmc9mwg4cl24hvzq5sfkv45ef9ahnyrr6rnr2vr63tzl5l3jpy950z7")
+		if err != nil {
+			panic(err)
+		}
+
+		// let's send the request to the scanner after 10sec
+		time.Sleep(time.Second * 3)
+		err = scanSvc.Watch(
+			scanner.WithStartBlock(1),
+			scanner.WithWatchItem(watchItem),
+		)
+		if err != nil {
+			panic(err)
+		}
+
+		signalQuit := make(chan os.Signal, 1)
+		signal.Notify(signalQuit, os.Interrupt)
+		<-signalQuit
+		node.Stop()
+		scanSvc.Stop()
+		return nil
+	}
+}

go.mod

@@ -3,9 +3,12 @@ module github.com/vulpemventures/neutrino-elements
 go 1.16
 
 require (
-	github.com/btcsuite/btcd v0.22.0-beta // indirect
+	github.com/btcsuite/btcd v0.22.0-beta
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
-	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd
+	github.com/google/go-cmp v0.5.6
+	github.com/sirupsen/logrus v1.8.1
+	github.com/urfave/cli/v2 v2.3.0
 	github.com/vulpemventures/go-elements v0.3.6
 	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
@@ -11,7 +12,6 @@ github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3L
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ=
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
 github.com/btcsuite/btcutil/psbt v1.0.2/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
-github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
 github.com/btcsuite/goleveldb v1.0.0 h1:Tvd0BfvqX9o823q1j2UZ/epQo09eJh6dTcRp79ilIN4=
@@ -21,13 +21,17 @@ github.com/btcsuite/snappy-go v1.0.0 h1:ZxaA6lo2EpxGddsA8JwWOcxlzRybb444sgmeJQMJ
 github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -40,10 +44,19 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/vulpemventures/fastsha256 v0.0.0-20160815193821-637e65642941 h1:CTcw80hz/Sw8hqlKX5ZYvBUF5gAHSHwdjXxRf/cjDcI=
 github.com/vulpemventures/fastsha256 v0.0.0-20160815193821-637e65642941/go.mod h1:GXBJykxW2kUcktGdsgyay7uwwWvkljASfljNcT0mbh8=
 github.com/vulpemventures/go-elements v0.3.6 h1:uS69KDTP6JTvrZRvqR2j7sUM4H1moQpdHTarew0kC7c=
@@ -67,16 +80,20 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/binary/marshaler.go

@@ -0,0 +1,129 @@
+package binary
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"reflect"
+)
+
+const (
+	commandLength          = 12
+	hashLength             = 32
+	magicAndChecksumLength = 4
+)
+
+// Marshaler is the interface implemented by types that can marshal themselves into binary.
+type Marshaler interface {
+	MarshalBinary() ([]byte, error)
+}
+
+// ensures that littleEndian is used to encode varint value
+func MarshalForVarint(i interface{}) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+	switch val := i.(type) {
+	case uint8, uint16, uint32, uint64:
+		if err := binary.Write(buf, binary.LittleEndian, val); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+
+	return nil, fmt.Errorf("unsuported varint value type")
+}
+
+// Marshal returns the binary encoding of v.
+func Marshal(v interface{}) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+
+	if reflect.TypeOf(v).Kind() == reflect.Ptr {
+		v = reflect.ValueOf(v).Elem().Interface()
+	}
+
+	switch val := v.(type) {
+	case uint8, int32, uint32, int64, uint64, bool:
+		if err := binary.Write(buf, binary.LittleEndian, val); err != nil {
+			return nil, err
+		}
+
+	// port
+	case uint16:
+		if err := binary.Write(buf, binary.BigEndian, val); err != nil {
+			return nil, err
+		}
+
+	case [magicAndChecksumLength]byte:
+		if _, err := buf.Write(val[:]); err != nil {
+			return nil, err
+		}
+
+	case [commandLength]byte:
+		if _, err := buf.Write(val[:]); err != nil {
+			return nil, err
+		}
+
+	case [hashLength]byte:
+		if _, err := buf.Write(val[:]); err != nil {
+			return nil, err
+		}
+
+	case []byte:
+		if _, err := buf.Write(val); err != nil {
+			return nil, err
+		}
+
+	// VarStr.String
+	case string:
+		if _, err := buf.Write([]byte(val)); err != nil {
+			return nil, err
+		}
+
+	case Marshaler:
+		b, err := val.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+
+		if _, err := buf.Write(b); err != nil {
+			return nil, err
+		}
+
+	default:
+		// is it a struct?
+		if reflect.ValueOf(v).Kind() == reflect.Struct {
+			b, err := marshalStruct(v)
+			if err != nil {
+				return nil, err
+			}
+
+			if _, err := buf.Write(b); err != nil {
+				return nil, err
+			}
+
+			break
+		}
+
+		return nil, fmt.Errorf("unsupported type %s", reflect.TypeOf(v).String())
+	}
+
+	return buf.Bytes(), nil
+}
+
+func marshalStruct(v interface{}) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+	vv := reflect.ValueOf(v)
+
+	for i := 0; i < vv.NumField(); i++ {
+		s, err := Marshal(reflect.Indirect(vv.Field(i)).Interface())
+		if err != nil {
+			f := reflect.TypeOf(v).Field(i).Name
+			return nil, fmt.Errorf("failed to marshal field %s: %v", f, err)
+		}
+
+		if _, err := buf.Write(s); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}