dcrdata v6 module reorganization and treasury TODOs

This nukes most of the old dcrdata modules and reorganizes large portions
of the source. There are now 4 modules that are imported by another
cmd/dcrdata module:

 - root module
 - db/dcrpg module
 - gov module
 - exchanges module

The dcrdata main package is moved into cmd/dcrdata, and several packages
only usable by the app (middleware, explorer, api (but not api/types), etc.)
are put under that module as well.

This also has some basic but incomplete handing for the three new treasury
types, and lots of TODOs. The inputs and outputs of the Treasurybase,
Treasury Spend, and Treasury Add txn types are not handled properly yet.
The /block page also needs 3 new txn categories.
The new vote version with a final nulldata output (e.g. https://testnet.dcrdata.org/tx/abde4c69df11cd0bce45a1ad729801afd75b270a05f55ae68defc3f910ab8c02)
should also have special handling, such as deciding the treasury vote's
position on a particular tspend.

.github/workflows/build.yml

@@ -19,16 +19,19 @@ jobs:
     - name: Go Build
       env:
         GO111MODULE: "on"
-      run: go build ./...
+      run: go build -v ./...
+      working-directory: ./cmd/dcrdata
 
     - name: Use nodejs ${{ matrix.node-version }}
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
     - name: npm install
       run: npm install
+      working-directory: ./cmd/dcrdata
     - name: npm run build
       run: npm run build
+      working-directory: ./cmd/dcrdata
 
     - name: Install Linters
       run: "curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.32.0"

.github/workflows/docker.yml

@@ -5,6 +5,6 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Build the Docker image
       run: docker build -t decred/dcrdata:$(date +%s) .

.gitignore

@@ -4,10 +4,11 @@
 *.log.*.gz
 *.bak
 .idea/
+/dcrdata
 debug
 *.cert
 *.key
-dcrdata
+cmd/dcrdata/dcrdata
 dcrdata.conf
 /vendor/
 *.orig
@@ -42,16 +43,19 @@ v5
 tmp/runner-build
 node_modules/
 public/dist/
+cmd/dcrdata/public/dist/
 public/js/dist/
+cmd/dcrdata/public/js/dist/
 public/**/*.gz
+cmd/dcrdata/public/**/*.gz
 testutil/apiload/results/
 testutil/apiload/apiload.conf
 testutil/apiload/apiload
 testutil/dbload/dbload
 testutil/dbconfig/test.data
 pubsub/democlient/democlient
-dcrrates/rateserver/rateserver
-dcrrates/rateserver/logs/
+exchanges/rateserver/rateserver
+exchanges/rateserver/logs/
 *.gob
 pgsql_*.sql
 sqlite_*.sql

Dockerfile

@@ -1,20 +1,20 @@
 FROM golang:1.15 as daemon
 
 COPY . /go/src
-WORKDIR /go/src
-RUN env GO111MODULE=on go build
+WORKDIR /go/src/cmd/dcrdata
+RUN env GO111MODULE=on go build -v
 
 FROM node:lts as gui
 
 WORKDIR /root
-COPY . /root
+COPY ./cmd/dcrdata /root
 RUN npm install
 RUN npm run build
 
 FROM golang:1.15
 WORKDIR /
-COPY --from=daemon /go/src/dcrdata /dcrdata
-COPY --from=daemon /go/src/views /views
+COPY --from=daemon /go/src/cmd/dcrdata/dcrdata /dcrdata
+COPY --from=daemon /go/src/cmd/dcrdata/views /views
 COPY --from=gui /root/public /public
 
 EXPOSE 7777

api/types/apicache.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2019, The Decred developers
+// Copyright (c) 2018-2020, The Decred developers
 // Copyright (c) 2017, Jonathan Chappelow
 // See LICENSE for details.
 

api/types/apitypes.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, The Decred developers
+// Copyright (c) 2019-2020, The Decred developers
 // Copyright (c) 2017, Jonathan Chappelow
 // See LICENSE for details.
 
@@ -12,8 +12,8 @@ import (
 	"time"
 
 	chainjson "github.com/decred/dcrd/rpc/jsonrpc/types/v2"
-	"github.com/decred/dcrdata/db/dbtypes/v2"
-	"github.com/decred/dcrdata/txhelpers/v4"
+	"github.com/decred/dcrdata/v6/db/dbtypes"
+	"github.com/decred/dcrdata/v6/txhelpers"
 )
 
 // TimeAPI is a fall back dbtypes.TimeDef wrapper that allows API endpoints that

api/types/insightapitypes.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2019, The Decred developers
+// Copyright (c) 2018-2020, The Decred developers
 // Copyright (c) 2017, Jonathan Chappelow
 // See LICENSE for details.
 
@@ -7,8 +7,8 @@ package types
 import (
 	"encoding/json"
 
-	"github.com/decred/dcrd/dcrutil/v2"
-	"github.com/decred/dcrdata/db/dbtypes/v2"
+	"github.com/decred/dcrd/dcrutil/v3"
+	"github.com/decred/dcrdata/v6/db/dbtypes"
 )
 
 // SyncResponse contains sync status information.
@@ -122,21 +122,22 @@ type SpendByFundingHash struct {
 }
 
 type InsightTx struct {
-	Txid          string         `json:"txid,omitempty"`
-	Version       int32          `json:"version,omitempty"`
-	Locktime      uint32         `json:"locktime"`
-	IsCoinBase    bool           `json:"isCoinBase,omitempty"`
-	Vins          []*InsightVin  `json:"vin,omitempty"`
-	Vouts         []*InsightVout `json:"vout,omitempty"`
-	Blockhash     string         `json:"blockhash,omitempty"`
-	Blockheight   int64          `json:"blockheight"`
-	Confirmations int64          `json:"confirmations"`
-	Time          int64          `json:"time,omitempty"`
-	Blocktime     int64          `json:"blocktime,omitempty"`
-	ValueOut      float64        `json:"valueOut,omitempty"`
-	Size          uint32         `json:"size,omitempty"`
-	ValueIn       float64        `json:"valueIn,omitempty"`
-	Fees          float64        `json:"fees,omitempty"`
+	Txid           string         `json:"txid,omitempty"`
+	Version        int32          `json:"version,omitempty"`
+	Locktime       uint32         `json:"locktime"`
+	IsCoinBase     bool           `json:"isCoinBase,omitempty"`
+	IsTreasurybase bool           `json:"isTreasurybase,omitempty"`
+	Vins           []*InsightVin  `json:"vin,omitempty"`
+	Vouts          []*InsightVout `json:"vout,omitempty"`
+	Blockhash      string         `json:"blockhash,omitempty"`
+	Blockheight    int64          `json:"blockheight"`
+	Confirmations  int64          `json:"confirmations"`
+	Time           int64          `json:"time,omitempty"`
+	Blocktime      int64          `json:"blocktime,omitempty"`
+	ValueOut       float64        `json:"valueOut,omitempty"`
+	Size           uint32         `json:"size,omitempty"`
+	ValueIn        float64        `json:"valueIn,omitempty"`
+	Fees           float64        `json:"fees,omitempty"`
 }
 
 type InsightVin struct {

blockdata/blockdata.go

@@ -1,24 +1,27 @@
+// Copyright (c) 2020, The Decred developers
 // Copyright (c) 2017, Jonathan Chappelow
 // See LICENSE for details.
 
 package blockdata
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
 	"time"
 
 	"github.com/decred/dcrd/chaincfg/chainhash"
-	"github.com/decred/dcrd/chaincfg/v2"
-	"github.com/decred/dcrd/dcrutil/v2"
+	"github.com/decred/dcrd/chaincfg/v3"
+	"github.com/decred/dcrd/dcrutil/v3"
 	chainjson "github.com/decred/dcrd/rpc/jsonrpc/types/v2"
-	"github.com/decred/dcrd/rpcclient/v5"
+	"github.com/decred/dcrd/rpcclient/v6"
 	"github.com/decred/dcrd/wire"
-	apitypes "github.com/decred/dcrdata/api/types/v5"
-	"github.com/decred/dcrdata/db/dbtypes/v2"
-	"github.com/decred/dcrdata/stakedb/v3"
-	"github.com/decred/dcrdata/txhelpers/v4"
+
+	apitypes "github.com/decred/dcrdata/v6/api/types"
+	"github.com/decred/dcrdata/v6/db/dbtypes"
+	"github.com/decred/dcrdata/v6/stakedb"
+	"github.com/decred/dcrdata/v6/txhelpers"
 )
 
 // BlockData contains all the data collected by a Collector and stored
@@ -147,7 +150,7 @@ func (t *Collector) CollectBlockInfo(hash *chainhash.Hash) (*apitypes.BlockDataB
 	*chainjson.FeeInfoBlock, *chainjson.GetBlockHeaderVerboseResult,
 	*apitypes.BlockExplorerExtraInfo, *wire.MsgBlock, error) {
 	// Retrieve block from dcrd.
-	msgBlock, err := t.dcrdChainSvr.GetBlock(hash)
+	msgBlock, err := t.dcrdChainSvr.GetBlock(context.TODO(), hash)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -157,17 +160,17 @@ func (t *Collector) CollectBlockInfo(hash *chainhash.Hash) (*apitypes.BlockDataB
 
 	// Coin supply and block subsidy. If either RPC fails, do not immediately
 	// return. Attempt acquisition of other data for this block.
-	coinSupply, err := t.dcrdChainSvr.GetCoinSupply()
+	coinSupply, err := t.dcrdChainSvr.GetCoinSupply(context.TODO())
 	if err != nil {
 		log.Error("GetCoinSupply failed: ", err)
 	}
-	nbSubsidy, err := t.dcrdChainSvr.GetBlockSubsidy(int64(msgBlock.Header.Height)+1, 5)
+	nbSubsidy, err := t.dcrdChainSvr.GetBlockSubsidy(context.TODO(), int64(msgBlock.Header.Height)+1, 5)
 	if err != nil {
 		log.Errorf("GetBlockSubsidy for %d failed: %v", msgBlock.Header.Height, err)
 	}
 
 	// Block header
-	blockHeaderResults, err := t.dcrdChainSvr.GetBlockHeaderVerbose(hash)
+	blockHeaderResults, err := t.dcrdChainSvr.GetBlockHeaderVerbose(context.TODO(), hash)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -241,14 +244,14 @@ func (t *Collector) CollectHash(hash *chainhash.Hash) (*BlockData, *wire.MsgBloc
 	}
 
 	// Number of peer connection to chain server
-	numConn, err := t.dcrdChainSvr.GetConnectionCount()
+	numConn, err := t.dcrdChainSvr.GetConnectionCount(context.TODO())
 	if err != nil {
 		log.Warn("Unable to get connection count: ", err)
 	}
 
 	// Blockchain info (e.g. syncheight, verificationprogress, chainwork,
 	// bestblockhash, initialblockdownload, maxblocksize, deployments, etc.).
-	chainInfo, err := t.dcrdChainSvr.GetBlockChainInfo()
+	chainInfo, err := t.dcrdChainSvr.GetBlockChainInfo(context.TODO())
 	if err != nil {
 		log.Warn("Unable to get blockchain info: ", err)
 	}
@@ -299,7 +302,7 @@ func (t *Collector) Collect() (*BlockData, *wire.MsgBlock, error) {
 	// verificationprogress, chainwork, bestblockhash, initialblockdownload,
 	// maxblocksize, deployments, etc.).
 	go func() {
-		blockchainInfo, err := t.dcrdChainSvr.GetBlockChainInfo()
+		blockchainInfo, err := t.dcrdChainSvr.GetBlockChainInfo(context.TODO())
 		toch <- bciRes{err, blockchainInfo}
 	}()
 
@@ -322,13 +325,13 @@ func (t *Collector) Collect() (*BlockData, *wire.MsgBlock, error) {
 	}
 
 	// Stake difficulty
-	stakeDiff, err := t.dcrdChainSvr.GetStakeDifficulty()
+	stakeDiff, err := t.dcrdChainSvr.GetStakeDifficulty(context.TODO())
 	if err != nil {
 		return nil, nil, err
 	}
 
 	// estimatestakediff
-	estStakeDiff, err := t.dcrdChainSvr.EstimateStakeDiff(nil)
+	estStakeDiff, err := t.dcrdChainSvr.EstimateStakeDiff(context.TODO(), nil)
 	if err != nil {
 		log.Warn("estimatestakediff is broken: ", err)
 		estStakeDiff = &chainjson.EstimateStakeDiffResult{}
@@ -342,7 +345,7 @@ func (t *Collector) Collect() (*BlockData, *wire.MsgBlock, error) {
 	}
 
 	// Number of peer connection to chain server
-	numConn, err := t.dcrdChainSvr.GetConnectionCount()
+	numConn, err := t.dcrdChainSvr.GetConnectionCount(context.TODO())
 	if err != nil {
 		log.Warn("Unable to get connection count: ", err)
 	}