tx/address indexing as optional feature, with separate db

cmd/geth/build_atxi_cmd.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"gopkg.in/urfave/cli.v1"
+	"strconv"
+	"github.com/ethereumproject/go-ethereum/logger/glog"
+	"github.com/ethereumproject/go-ethereum/core/types"
+	"path/filepath"
+	"io/ioutil"
+	"os"
+	"github.com/ethereumproject/go-ethereum/logger"
+	"time"
+)
+
+func buildAddrTxIndexCmd(ctx *cli.Context) error {
+	startIndex := uint64(ctx.Int("start"))
+	var stopIndex uint64
+
+	// Use persistent placeholder in case start not spec'd
+	placeholderFilename := filepath.Join(MustMakeChainDataDir(ctx), "index.at")
+	if !ctx.IsSet("start") {
+		bs, err := ioutil.ReadFile(placeholderFilename)
+		if err == nil { // ignore errors for now
+			startIndex, _ = strconv.ParseUint(string(bs), 10, 64)
+		}
+	}
+
+	bc, chainDB := MakeChain(ctx)
+	if bc == nil || chainDB == nil {
+		panic("bc or cdb is nil")
+	}
+	defer chainDB.Close()
+
+	stopIndex = uint64(ctx.Int("stop"))
+	if stopIndex == 0 {
+		stopIndex = bc.CurrentHeader().Number.Uint64()
+	}
+
+	if stopIndex < startIndex {
+		glog.Fatal("start must be prior to (smaller than) or equal to stop, got start=", startIndex, "stop=", stopIndex)
+	}
+	if startIndex == stopIndex {
+		glog.D(logger.Error).Infoln("Up to date. Exiting.")
+		os.Exit(0)
+	}
+
+	indexDb := MakeIndexDatabase(ctx)
+	if indexDb == nil {
+		panic("indexdb is nil")
+	}
+	defer indexDb.Close()
+
+	var block *types.Block
+	blockIndex := startIndex
+	block = bc.GetBlockByNumber(blockIndex)
+	if block == nil {
+		glog.Fatal(blockIndex, "block is nil")
+	}
+
+	var inc = uint64(ctx.Int("step"))
+	startTime := time.Now()
+	glog.D(logger.Error).Infoln("Address/tx indexing (atxi) start:", startIndex, "stop:", stopIndex, "step:", inc)
+	for i := startIndex; i <= stopIndex; i = i+inc {
+		if i+inc > stopIndex {
+			inc = stopIndex - startIndex
+		}
+		if err := bc.AddTxIndexesBatch(indexDb, i, i+inc); err != nil {
+			return err
+		}
+		ioutil.WriteFile(placeholderFilename, []byte(strconv.Itoa(int(i+inc))), os.ModePerm)
+	}
+	ioutil.WriteFile(placeholderFilename, []byte(strconv.Itoa(int(stopIndex))), os.ModePerm)
+	glog.D(logger.Error).Infoln("Finished atxi. Took:", time.Since(startTime))
+	return nil
+}
+
+

cmd/geth/flag.go

@@ -599,16 +599,17 @@ func mustMakeEthConf(ctx *cli.Context, sconf *core.SufficientChainConfig) *eth.C
 	}
 
 	ethConf := &eth.Config{
-		ChainConfig:             sconf.ChainConfig,
-		Genesis:                 sconf.Genesis,
-		FastSync:                ctx.GlobalBool(aliasableName(FastSyncFlag.Name, ctx)),
-		BlockChainVersion:       ctx.GlobalInt(aliasableName(BlockchainVersionFlag.Name, ctx)),
-		DatabaseCache:           ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)),
-		DatabaseHandles:         MakeDatabaseHandles(),
-		NetworkId:               sconf.Network,
-		AccountManager:          accman,
-		Etherbase:               MakeEtherbase(accman, ctx),
-		MinerThreads:            ctx.GlobalInt(aliasableName(MinerThreadsFlag.Name, ctx)),
+		ChainConfig:       sconf.ChainConfig,
+		Genesis:           sconf.Genesis,
+		UseAddrTxIndex:    ctx.GlobalBool(aliasableName(AddrTxIndexFlag.Name, ctx)),
+		FastSync:          ctx.GlobalBool(aliasableName(FastSyncFlag.Name, ctx)),
+		BlockChainVersion: ctx.GlobalInt(aliasableName(BlockchainVersionFlag.Name, ctx)),
+		DatabaseCache:     ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx)),
+		DatabaseHandles:   MakeDatabaseHandles(),
+		NetworkId:         sconf.Network,
+		AccountManager:    accman,
+		Etherbase:         MakeEtherbase(accman, ctx),
+		MinerThreads:      ctx.GlobalInt(aliasableName(MinerThreadsFlag.Name, ctx)),
 		NatSpec:                 ctx.GlobalBool(aliasableName(NatspecEnabledFlag.Name, ctx)),
 		DocRoot:                 ctx.GlobalString(aliasableName(DocRootFlag.Name, ctx)),
 		GasPrice:                new(big.Int),
@@ -799,18 +800,32 @@ func MustMakeChainConfigFromDefaults(ctx *cli.Context) *core.ChainConfig {
 // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
 func MakeChainDatabase(ctx *cli.Context) ethdb.Database {
 	var (
-		datadir = MustMakeChainDataDir(ctx)
-		cache   = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
-		handles = MakeDatabaseHandles()
+		chaindir = MustMakeChainDataDir(ctx)
+		cache    = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
+		handles  = MakeDatabaseHandles()
 	)
 
-	chainDb, err := ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles)
+	chainDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "chaindata"), cache, handles)
 	if err != nil {
 		glog.Fatal("Could not open database: ", err)
 	}
 	return chainDb
 }
 
+func MakeIndexDatabase(ctx *cli.Context) ethdb.Database {
+	var (
+		chaindir = MustMakeChainDataDir(ctx)
+		cache    = ctx.GlobalInt(aliasableName(CacheFlag.Name, ctx))
+		handles  = MakeDatabaseHandles()
+	)
+
+	indexesDb, err := ethdb.NewLDBDatabase(filepath.Join(chaindir, "indexes"), cache, handles)
+	if err != nil {
+		glog.Fatal("Could not open database: ", err)
+	}
+	return indexesDb
+}
+
 // MakeChain creates a chain manager from set command line flags.
 func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) {
 	var err error

cmd/geth/flags.go

@@ -89,6 +89,10 @@ var (
 		Name:  "light-kdf,lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	AddrTxIndexFlag = cli.BoolFlag{
+		Name: "add-tx-index,atxi",
+		Usage: "Toggle indexes for transactions by address. Pre-existing chaindata can be indexed with command 'geth atxi-build'",
+	}
 	// Network Split settings
 	ETFChain = cli.BoolFlag{
 		Name:  "etf",

cmd/geth/main.go

@@ -99,6 +99,36 @@ func makeCLIApp() (app *cli.App) {
 	The output of this command is supposed to be machine-readable.
 			`,
 		},
+		{
+			Action: buildAddrTxIndexCmd,
+			Name:   "atxi-build",
+			Usage:  "Generate index for transactions by address",
+			Description: `
+	Builds an index for transactions by address. 
+	The command is idempotent; it will not hurt to run multiple times on the same range.
+
+	If run without --start flag, the command makes use of a persistent placeholder, so you can
+	run 'geth build-atxi' on multiple occasions and pick up indexing progress where the last session
+	left off.
+
+	To enable address-transaction indexing during block sync and import, use the '--atxi' flag.
+			`,
+			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "start",
+					Usage: "Block number at which to begin building index",
+				},
+				cli.IntFlag{
+					Name:  "stop",
+					Usage: "Block number at which to stop building index",
+				},
+				cli.IntFlag{
+					Name: "step",
+					Usage: "Step increment for batching. Higher number requires more mem, but may be faster. [default: 10000]",
+					Value: 10000,
+				},
+			},
+		},
 		{
 			Action: makeMLogDocumentation,
 			Name:   "mdoc",
@@ -132,6 +162,7 @@ func makeCLIApp() (app *cli.App) {
 		ChainIdentityFlag,
 		BlockchainVersionFlag,
 		FastSyncFlag,
+		AddrTxIndexFlag,
 		CacheFlag,
 		LightKDFFlag,
 		JSpathFlag,