[gov] Migrate to new politeia tlog API

cmd/dcrdata/api/apiroutes.go

@@ -33,6 +33,7 @@ import (
 	m "github.com/decred/dcrdata/cmd/dcrdata/middleware"
 	"github.com/decred/dcrdata/exchanges/v3"
 	"github.com/decred/dcrdata/gov/v4/agendas"
+	"github.com/decred/dcrdata/gov/v4/politeia"
 	apitypes "github.com/decred/dcrdata/v6/api/types"
 	"github.com/decred/dcrdata/v6/db/cache"
 	"github.com/decred/dcrdata/v6/db/dbtypes"
@@ -70,7 +71,6 @@ type DataSource interface {
 	Height() int64
 	AllAgendas() (map[string]dbtypes.MileStone, error)
 	GetTicketInfo(txid string) (*apitypes.TicketInfo, error)
-	ProposalVotes(proposalToken string) (*dbtypes.ProposalChartsData, error)
 	PowerlessTickets() (*apitypes.PowerlessTickets, error)
 	GetStakeInfoExtendedByHash(hash string) *apitypes.StakeInfoExtended
 	GetStakeInfoExtendedByHeight(idx int) *apitypes.StakeInfoExtended
@@ -111,29 +111,29 @@ type DataSource interface {
 
 // dcrdata application context used by all route handlers
 type appContext struct {
-	nodeClient   *rpcclient.Client
-	Params       *chaincfg.Params
-	DataSource   DataSource
-	Status       *apitypes.Status
-	xcBot        *exchanges.ExchangeBot
-	AgendaDB     *agendas.AgendaDB
-	maxCSVAddrs  int
-	charts       *cache.ChartData
-	isPiDisabled bool // is piparser disabled
+	nodeClient  *rpcclient.Client
+	Params      *chaincfg.Params
+	DataSource  DataSource
+	Status      *apitypes.Status
+	xcBot       *exchanges.ExchangeBot
+	AgendaDB    *agendas.AgendaDB
+	ProposalsDB *politeia.ProposalsDB
+	maxCSVAddrs int
+	charts      *cache.ChartData
 }
 
 // AppContextConfig is the configuration for the appContext and the only
 // argument to its constructor.
 type AppContextConfig struct {
-	Client             *rpcclient.Client
-	Params             *chaincfg.Params
-	DataSource         DataSource
-	XcBot              *exchanges.ExchangeBot
-	AgendasDBInstance  *agendas.AgendaDB
-	MaxAddrs           int
-	Charts             *cache.ChartData
-	IsPiparserDisabled bool
-	AppVer             string
+	Client            *rpcclient.Client
+	Params            *chaincfg.Params
+	DataSource        DataSource
+	XcBot             *exchanges.ExchangeBot
+	AgendasDBInstance *agendas.AgendaDB
+	ProposalsDB       *politeia.ProposalsDB
+	MaxAddrs          int
+	Charts            *cache.ChartData
+	AppVer            string
 }
 
 // NewContext constructs a new appContext from the RPC client and database, and
@@ -149,15 +149,15 @@ func NewContext(cfg *AppContextConfig) *appContext {
 	}
 
 	return &appContext{
-		nodeClient:   cfg.Client,
-		Params:       cfg.Params,
-		DataSource:   cfg.DataSource,
-		xcBot:        cfg.XcBot,
-		AgendaDB:     cfg.AgendasDBInstance,
-		Status:       apitypes.NewStatus(uint32(nodeHeight), conns, APIVersion, cfg.AppVer, cfg.Params.Name),
-		maxCSVAddrs:  cfg.MaxAddrs,
-		charts:       cfg.Charts,
-		isPiDisabled: cfg.IsPiparserDisabled,
+		nodeClient:  cfg.Client,
+		Params:      cfg.Params,
+		DataSource:  cfg.DataSource,
+		xcBot:       cfg.XcBot,
+		AgendaDB:    cfg.AgendasDBInstance,
+		ProposalsDB: cfg.ProposalsDB,
+		Status:      apitypes.NewStatus(uint32(nodeHeight), conns, APIVersion, cfg.AppVer, cfg.Params.Name),
+		maxCSVAddrs: cfg.MaxAddrs,
+		charts:      cfg.Charts,
 	}
 }
 
@@ -1200,28 +1200,22 @@ func (c *appContext) getTicketPoolByDate(w http.ResponseWriter, r *http.Request)
 }
 
 func (c *appContext) getProposalChartData(w http.ResponseWriter, r *http.Request) {
-	if c.isPiDisabled {
-		errMsg := "piparser is disabled."
-		apiLog.Errorf("%s. Remove the disable-piparser flag to activate it.", errMsg)
-		http.Error(w, errMsg, http.StatusServiceUnavailable)
-		return
-	}
-
 	token := m.GetProposalTokenCtx(r)
-	votesData, err := c.DataSource.ProposalVotes(token)
+
+	proposal, err := c.ProposalsDB.ProposalByToken(token)
 	if dbtypes.IsTimeoutErr(err) {
-		apiLog.Errorf("ProposalVotes: %v", err)
+		apiLog.Errorf("ProposalByToken: %v", err)
 		http.Error(w, "Database timeout.", http.StatusServiceUnavailable)
 		return
 	}
 	if err != nil {
-		apiLog.Errorf("Unable to get proposals votes for token %s : %v", token, err)
+		apiLog.Errorf("Unable to get proposal chart data for token %s : %v", token, err)
 		http.Error(w, http.StatusText(http.StatusUnprocessableEntity),
 			http.StatusUnprocessableEntity)
 		return
 	}
 
-	writeJSON(w, votesData, m.GetIndentCtx(r))
+	writeJSON(w, proposal.ChartData, m.GetIndentCtx(r))
 }
 
 func (c *appContext) getBlockSize(w http.ResponseWriter, r *http.Request) {

cmd/dcrdata/config.go

@@ -66,7 +66,7 @@ var (
 
 	defaultAgendasDBFileName = "agendas.db"
 	defaultProposalsFileName = "proposals.db"
-	defaultPoliteiaAPIURl    = "https://proposals.decred.org"
+	defaultPoliteiaURL       = "https://proposals.decred.org/"
 	defaultChartsCacheDump   = "chartscache.gob"
 
 	defaultPGHost           = "127.0.0.1:5432"
@@ -124,13 +124,10 @@ type config struct {
 	MempoolMaxInterval int `long:"mp-max-interval" description:"The maximum time in seconds between mempool reports (within a couple seconds), regardless of number of new tickets seen." env:"DCRDATA_MEMPOOL_MAX_INTERVAL"`
 	MPTriggerTickets   int `long:"mp-ticket-trigger" description:"The minimum number of new tickets that must be seen to trigger a new mempool report." env:"DCRDATA_MP_TRIGGER_TICKETS"`
 
-	// Politeia/proposals and consensus agendas
+	// Consensus agendas and politeia proposals
 	AgendasDBFileName string `long:"agendadbfile" description:"Agendas DB file name (default is agendas.db)." env:"DCRDATA_AGENDAS_DB_FILE_NAME"`
 	ProposalsFileName string `long:"proposalsdbfile" description:"Proposals DB file name (default is proposals.db)." env:"DCRDATA_PROPOSALS_DB_FILE_NAME"`
-	PoliteiaAPIURL    string `long:"politeiaurl" description:"Defines the root API politeia URL (defaults to https://proposals.decred.org)." env:"DCRDATA_POLITEIA_URL"`
-	PiPropRepoOwner   string `long:"piproposalsowner" description:"Defines the owner to the github repo where Politeia's proposals are pushed." env:"DCRDATA_PI_REPO_OWNER"`
-	PiPropRepoName    string `long:"piproposalsrepo" description:"Defines the name of the github repo where Politeia's proposals are pushed." env:"DCRDATA_PROPS_REPO"`
-	DisablePiParser   bool   `long:"disable-piparser" description:"Disables the piparser tool from running." env:"DCRDATA_DISABLE_PIPARSER"`
+	PoliteiaURL       string `long:"politeiaurl" description:"Defines the root API politeia URL (defaults to https://proposals.decred.org/)." env:"DCRDATA_POLITEIA_URL"`
 
 	// Caching and optimization.
 	AddrCacheCap     int    `long:"addr-cache-cap" description:"Address cache capacity in bytes." env:"DCRDATA_ADDR_CACHE_CAP"`
@@ -182,7 +179,7 @@ var (
 		ConfigFile:          defaultConfigFile,
 		AgendasDBFileName:   defaultAgendasDBFileName,
 		ProposalsFileName:   defaultProposalsFileName,
-		PoliteiaAPIURL:      defaultPoliteiaAPIURl,
+		PoliteiaURL:         defaultPoliteiaURL,
 		ChartsCacheDump:     defaultChartsCacheDump,
 		DebugLevel:          defaultLogLevel,
 		HTTPProfPath:        defaultHTTPProfPath,
@@ -526,8 +523,6 @@ func loadConfig() (*config, error) {
 		activeNet = &netparams.SimNetParams
 		activeChain = chaincfg.SimNetParams()
 		defaultPort = defaultSimnetPort
-		// If on simnet, disable piparser tool automatically.
-		cfg.DisablePiParser = true
 		numNets++
 	}
 	if numNets > 1 {
@@ -636,14 +631,6 @@ func loadConfig() (*config, error) {
 		return loadConfigError(err)
 	}
 
-	// Checks if the expected format of the API URL was set. It also drops any
-	// unnecessary parts of the URL.
-	urlPath, err := retrieveRootPath(cfg.PoliteiaAPIURL)
-	if err != nil {
-		return loadConfigError(err)
-	}
-	cfg.PoliteiaAPIURL = urlPath
-
 	// Check the supplied APIListen address
 	if cfg.APIListen == "" {
 		cfg.APIListen = defaultHost + ":" + defaultPort
@@ -654,6 +641,14 @@ func loadConfig() (*config, error) {
 		}
 	}
 
+	// Checks if the expected format of the politeia URL was set. It also drops any
+	// unnecessary parts of the URL.
+	urlPath, err := retrieveRootPath(cfg.PoliteiaURL)
+	if err != nil {
+		return loadConfigError(err)
+	}
+	cfg.PoliteiaURL = urlPath
+
 	switch cfg.ServerHeader {
 	case "off":
 		cfg.ServerHeader = ""

cmd/dcrdata/config_test.go

@@ -235,19 +235,6 @@ func TestDefaultConfigTestNetWithEnvAndBadValue(t *testing.T) {
 	}
 }
 
-func TestDisablePiparserValueOnSimnet(t *testing.T) {
-	os.Args = append(os.Args, "--simnet")
-
-	cfg, err := loadConfig()
-	if err != nil {
-		t.Errorf("expected to find no error but found: %v", err)
-	}
-
-	if !cfg.DisablePiParser {
-		t.Fatal("expected DisablePiParser value to be activated but it wasn't")
-	}
-}
-
 func TestRetrieveRootPath(t *testing.T) {
 	type testData struct {
 		RawURL   string

cmd/dcrdata/explorer/explorer.go

@@ -95,7 +95,6 @@ type explorerDataSource interface {
 	TimeBasedIntervals(timeGrouping dbtypes.TimeBasedGrouping, limit, offset uint64) ([]*dbtypes.BlocksGroupedInfo, error)
 	AgendasVotesSummary(agendaID string) (summary *dbtypes.AgendaSummary, err error)
 	BlockTimeByHeight(height int64) (int64, error)
-	LastPiParserSync() time.Time
 	GetChainParams() *chaincfg.Params
 	GetExplorerBlock(hash string) *types.BlockInfo
 	GetExplorerBlocks(start int, end int) []*types.BlockBasic
@@ -114,13 +113,11 @@ type explorerDataSource interface {
 	Difficulty(timestamp int64) float64
 }
 
-// PoliteiaBackend implements methods that manage proposals db data.
 type PoliteiaBackend interface {
-	LastProposalsSync() int64
-	CheckProposalsUpdates() error
-	AllProposals(offset, rowsCount int, filterByVoteStatus ...int) (proposals []*pitypes.ProposalInfo, totalCount int, err error)
-	ProposalByToken(proposalToken string) (*pitypes.ProposalInfo, error)
-	ProposalByRefID(RefID string) (*pitypes.ProposalInfo, error)
+	ProposalsLastSync() int64
+	ProposalsSync() error
+	ProposalsAll(offset, rowsCount int, filterByVoteStatus ...int) ([]*pitypes.ProposalRecord, int, error)
+	ProposalByToken(token string) (*pitypes.ProposalRecord, error)
 }
 
 // agendaBackend implements methods that manage agendas db data.
@@ -208,7 +205,7 @@ type explorerUI struct {
 	chartSource      ChartDataSource
 	agendasSource    agendaBackend
 	voteTracker      *agendas.VoteTracker
-	proposalsSource  PoliteiaBackend
+	proposals        PoliteiaBackend
 	dbsSyncing       atomic.Value
 	devPrefetch      bool
 	templates        templates
@@ -223,7 +220,7 @@ type explorerUI struct {
 	// displaySyncStatusPage indicates if the sync status page is the only web
 	// page that should be accessible during DB synchronization.
 	displaySyncStatusPage atomic.Value
-	politeiaAPIURL        string
+	politeiaURL           string
 
 	invsMtx sync.RWMutex
 	invs    *types.MempoolInfo
@@ -278,21 +275,21 @@ func (exp *explorerUI) StopWebsocketHub() {
 
 // ExplorerConfig is the configuration settings for explorerUI.
 type ExplorerConfig struct {
-	DataSource      explorerDataSource
-	ChartSource     ChartDataSource
-	UseRealIP       bool
-	AppVersion      string
-	DevPrefetch     bool
-	Viewsfolder     string
-	XcBot           *exchanges.ExchangeBot
-	AgendasSource   agendaBackend
-	Tracker         *agendas.VoteTracker
-	ProposalsSource PoliteiaBackend
-	PoliteiaURL     string
-	MainnetLink     string
-	TestnetLink     string
-	OnionAddress    string
-	ReloadHTML      bool
+	DataSource    explorerDataSource
+	ChartSource   ChartDataSource
+	UseRealIP     bool
+	AppVersion    string
+	DevPrefetch   bool
+	Viewsfolder   string
+	XcBot         *exchanges.ExchangeBot
+	AgendasSource agendaBackend
+	Tracker       *agendas.VoteTracker
+	Proposals     PoliteiaBackend
+	PoliteiaURL   string
+	MainnetLink   string
+	TestnetLink   string
+	OnionAddress  string
+	ReloadHTML    bool
 }
 
 // New returns an initialized instance of explorerUI
@@ -309,8 +306,8 @@ func New(cfg *ExplorerConfig) *explorerUI {
 	exp.xcDone = make(chan struct{})
 	exp.agendasSource = cfg.AgendasSource
 	exp.voteTracker = cfg.Tracker
-	exp.proposalsSource = cfg.ProposalsSource
-	exp.politeiaAPIURL = cfg.PoliteiaURL
+	exp.proposals = cfg.Proposals
+	exp.politeiaURL = cfg.PoliteiaURL
 	explorerLinks.Mainnet = cfg.MainnetLink
 	explorerLinks.Testnet = cfg.TestnetLink
 	explorerLinks.MainnetSearch = cfg.MainnetLink + "search?search="
@@ -589,17 +586,15 @@ func (exp *explorerUI) Store(blockData *blockdata.BlockData, msgBlock *wire.MsgB
 		go exp.voteTracker.Refresh()
 	}
 
-	// Politeia updates happen hourly. Thus, if blocks take 5 minutes on average
-	// to mine, then 12 blocks take approximately 1hr.
-	// https://docs.decred.org/advanced/navigating-politeia-data/#voting-and-comment-data
-	if newBlockData.Height%12 == 0 && exp.proposalsSource != nil {
+	// Update proposals data every 5 blocks
+	if (newBlockData.Height%5 == 0) && exp.proposals != nil {
 		// Update the proposal DB. This is run asynchronously since it involves
 		// a query to Politeia (a remote system) and we do not want to block
 		// execution.
 		go func() {
-			err := exp.proposalsSource.CheckProposalsUpdates()
+			err := exp.proposals.ProposalsSync()
 			if err != nil {
-				log.Errorf("(PoliteiaBackend).CheckProposalsUpdates: %v", err)
+				log.Errorf("(PoliteiaBackend).ProposalsSync: %v", err)
 			}
 		}()
 	}

cmd/dcrdata/explorer/explorermiddleware.go

@@ -27,7 +27,7 @@ const (
 	ctxTxInOutId
 	ctxAddress
 	ctxAgendaId
-	ctxProposalRefID
+	ctxProposalToken
 )
 
 const (
@@ -144,7 +144,7 @@ func getAgendaIDCtx(r *http.Request) string {
 }
 
 func getProposalTokenCtx(r *http.Request) string {
-	hash, ok := r.Context().Value(ctxProposalRefID).(string)
+	hash, ok := r.Context().Value(ctxProposalToken).(string)
 	if !ok {
 		log.Trace("Proposal ref ID not set")
 		return ""
@@ -190,11 +190,11 @@ func AgendaPathCtx(next http.Handler) http.Handler {
 	})
 }
 
-// ProposalPathCtx embeds "proposalrefID" into the request context
+// ProposalPathCtx embeds "proposaltoken" into the request context
 func ProposalPathCtx(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		proposalRefID := chi.URLParam(r, "proposalrefid")
-		ctx := context.WithValue(r.Context(), ctxProposalRefID, proposalRefID)
+		proposalToken := chi.URLParam(r, "proposaltoken")
+		ctx := context.WithValue(r.Context(), ctxProposalToken, proposalToken)
 		next.ServeHTTP(w, r.WithContext(ctx))
 	})
 }