Move to using cobra.Command rather than built-in flag parsing

exp/lighthorizon/actions/root.go

@@ -0,0 +1,29 @@
+package actions
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/stellar/go/support/log"
+	supportProblem "github.com/stellar/go/support/render/problem"
+)
+
+type RootResponse struct {
+	Version      string `json:"version"`
+	LedgerSource string `json:"ledger_source"`
+	IndexSource  string `json:"index_source"`
+	LatestLedger uint32 `json:"latest_indexed_ledger"`
+}
+
+func Root(config RootResponse) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/hal+json; charset=utf-8")
+		encoder := json.NewEncoder(w)
+		encoder.SetIndent("", "  ")
+		err := encoder.Encode(config)
+		if err != nil {
+			log.Error(err)
+			sendErrorResponse(r.Context(), w, supportProblem.ServerError)
+		}
+	}
+}

exp/lighthorizon/main.go

@@ -1,80 +1,148 @@
 package main
 
 import (
-	"flag"
+	"context"
 	"net/http"
 
+	"github.com/go-chi/chi"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
 
+	"github.com/stellar/go/exp/lighthorizon/actions"
 	"github.com/stellar/go/exp/lighthorizon/index"
 	"github.com/stellar/go/exp/lighthorizon/ingester"
 	"github.com/stellar/go/exp/lighthorizon/services"
+	"github.com/stellar/go/exp/lighthorizon/tools"
 
 	"github.com/stellar/go/network"
+	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/support/log"
 )
 
 const (
-	defaultCacheSize = (60 * 60 * 24) / 6 // 1 day of ledgers @ 6s each
+	HorizonLiteVersion = "0.0.1-alpha"
+	defaultCacheSize   = (60 * 60 * 24) / 6 // 1 day of ledgers @ 6s each
 )
 
 func main() {
-	sourceUrl := flag.String("source", "gcs://horizon-archive-poc", "history archive url to read txmeta files")
-	indexesUrl := flag.String("indexes", "file://indexes", "url of the indexes")
-	networkPassphrase := flag.String("network-passphrase", network.PublicNetworkPassphrase, "network passphrase")
-	cacheDir := flag.String("ledger-cache", "", `path to cache frequently-used ledgers;
-if left empty, uses a temporary directory`)
-	cacheSize := flag.Int("ledger-cache-size", defaultCacheSize,
-		"number of ledgers to store in the cache")
-	logLevelParam := flag.String("log-level", "info",
-		"logging level, info, debug, warn, error, panic, fatal, trace, default is info")
-	flag.Parse()
-
-	L := log.WithField("service", "horizon-lite")
-	logLevel, err := logrus.ParseLevel(*logLevelParam)
-	if err != nil {
-		log.Warnf("Failed to parse -log-level '%s', defaulting to 'info'.", *logLevelParam)
-		logLevel = log.InfoLevel
-	}
-	L.SetLevel(logLevel)
-	L.Info("Starting lighthorizon!")
-
-	registry := prometheus.NewRegistry()
-	indexStore, err := index.ConnectWithConfig(index.StoreConfig{
-		URL:     *indexesUrl,
-		Log:     L.WithField("subservice", "index"),
-		Metrics: registry,
-	})
-	if err != nil {
-		panic(err)
+	cmd := &cobra.Command{
+		Use:   "lighthorizon <subcommand>",
+		Short: "Horizon Lite suite",
+		Long:  "Horizon Lite suite",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Usage()
+		},
 	}
 
-	ingester, err := ingester.NewIngester(ingester.IngesterConfig{
-		SourceUrl:         *sourceUrl,
-		NetworkPassphrase: *networkPassphrase,
-		CacheDir:          *cacheDir,
-		CacheSize:         *cacheSize,
-	})
-	if err != nil {
-		panic(err)
-	}
+	serve := &cobra.Command{
+		Use: "serve <txmeta source> <index source>",
+		Long: `Starts the Horizon Lite server, which only serves the following endpoint:
 
-	Config := services.Config{
-		Ingester:   ingester,
-		Passphrase: *networkPassphrase,
-		IndexStore: indexStore,
-		Metrics:    services.NewMetrics(registry),
-	}
+	GET /accounts/:id/transactions
 
-	lightHorizon := services.LightHorizon{
-		Transactions: &services.TransactionRepository{
-			Config: Config,
-		},
-		Operations: &services.OperationRepository{
-			Config: Config,
+The <txmeta source> should be a URL to meta archives from which to read unpacked
+ledger files, while the <index source> should be a URL containing indices that
+break down accounts by active ledgers.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) != 2 {
+				cmd.Usage()
+				return
+			}
+
+			sourceUrl, indexStoreUrl := args[0], args[1]
+
+			networkPassphrase, _ := cmd.Flags().GetString("network-passphrase")
+			switch networkPassphrase {
+			case "testnet":
+				networkPassphrase = network.TestNetworkPassphrase
+			case "pubnet":
+				networkPassphrase = network.PublicNetworkPassphrase
+			}
+
+			cacheDir, _ := cmd.Flags().GetString("ledger-cache")
+			cacheSize, _ := cmd.Flags().GetUint("ledger-cache-size")
+			parallelDL, _ := cmd.Flags().GetUint("parallel-downloads")
+			logLevelParam, _ := cmd.Flags().GetString("log-level")
+
+			L := log.WithField("service", "horizon-lite")
+			logLevel, err := logrus.ParseLevel(logLevelParam)
+			if err != nil {
+				log.Warnf("Failed to parse log level '%s', defaulting to 'info'.", logLevelParam)
+				logLevel = log.InfoLevel
+			}
+			L.SetLevel(logLevel)
+			L.Info("Starting lighthorizon!")
+
+			registry := prometheus.NewRegistry()
+			indexStore, err := index.ConnectWithConfig(index.StoreConfig{
+				URL:     indexStoreUrl,
+				Log:     L.WithField("service", "index"),
+				Metrics: registry,
+			})
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+
+			ingester, err := ingester.NewIngester(ingester.IngesterConfig{
+				SourceUrl:         sourceUrl,
+				NetworkPassphrase: networkPassphrase,
+				CacheDir:          cacheDir,
+				CacheSize:         int(cacheSize),
+				ParallelDownloads: parallelDL,
+			})
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+
+			Config := services.Config{
+				Ingester:   ingester,
+				Passphrase: networkPassphrase,
+				IndexStore: indexStore,
+				Metrics:    services.NewMetrics(registry),
+			}
+
+			lightHorizon := services.LightHorizon{
+				Transactions: &services.TransactionRepository{
+					Config: Config,
+				},
+				Operations: &services.OperationRepository{
+					Config: Config,
+				},
+			}
+
+			// Inject our config into the root response.
+			latestLedger, err := ingester.GetLatestLedgerSequence(context.Background())
+			if err != nil {
+				log.Fatal(errors.Wrapf(err, "failed to retrieve latest ledger"))
+			}
+
+			router := lightHorizonHTTPHandler(registry, lightHorizon).(*chi.Mux)
+			router.MethodFunc(http.MethodGet, "/", actions.Root(actions.RootResponse{
+				Version:      HorizonLiteVersion,
+				LedgerSource: sourceUrl,
+				IndexSource:  indexStoreUrl,
+				LatestLedger: latestLedger,
+			}))
+
+			log.Fatal(http.ListenAndServe(":8080", router))
 		},
 	}
 
-	log.Fatal(http.ListenAndServe(":8080", lightHorizonHTTPHandler(registry, lightHorizon)))
+	serve.Flags().String("log-level", "info",
+		"logging level: 'info', 'debug', 'warn', 'error', 'panic', 'fatal', or 'trace'")
+	serve.Flags().String("network-passphrase", "pubnet", "network passphrase")
+	serve.Flags().String("ledger-cache", "", "path to cache frequently-used ledgers; "+
+		"if left empty, uses a temporary directory")
+	serve.Flags().Uint("ledger-cache-size", defaultCacheSize,
+		"number of ledgers to store in the cache")
+	serve.Flags().Uint("parallel-downloads", 1,
+		"number of parallel ledger downloads to run at once")
+
+	cmd.AddCommand(serve)
+	tools.AddCacheCommands(cmd)
+	tools.AddIndexCommands(cmd)
+	cmd.Execute()
 }

exp/lighthorizon/services/main.go

@@ -17,8 +17,9 @@ import (
 )
 
 const (
-	allTransactionsIndex = "all/all"
-	allPaymentsIndex     = "all/payments"
+	allTransactionsIndex       = "all/all"
+	allPaymentsIndex           = "all/payments"
+	slowFetchDurationThreshold = time.Second
 )
 
 var (
@@ -122,11 +123,12 @@ func searchAccountTransactions(ctx context.Context,
 			return errors.Wrapf(ledgerErr,
 				"ledger export state is out of sync at ledger %d", nextLedger)
 		}
-		fetchDuration += time.Since(start)
-		if fetchDuration > time.Second {
-			log.WithField("duration", fetchDuration).
+		thisFetchDuration := time.Since(start)
+		if thisFetchDuration > slowFetchDurationThreshold {
+			log.WithField("duration", thisFetchDuration).
 				Warnf("Fetching ledger %d was really slow", nextLedger)
 		}
+		fetchDuration += thisFetchDuration
 
 		start = time.Now()
 		reader, readerErr := config.Ingester.NewLedgerTransactionReader(ledger)