politeiawww: Fix legacy pagination bug.

politeiawww/cmd/pictl/cmdlegacytest.go

@@ -10,57 +10,121 @@ import (
 	www "github.com/decred/politeia/politeiawww/api/www/v1"
 )
 
+// cmdLegacyTest tests the legacy www routes. These routes have been deprecated
+// and do not have corresponding pictl commands.
 type cmdLegacyTest struct {
+	Args struct {
+		AdminEmail    string `positional-arg-name:"adminemail" required:"true"`
+		AdminPassword string `positional-arg-name:"adminpassword" required:"true"`
+	} `positional-args:"true"`
 }
 
 // Execute executes the cmdLegacyTest command.
 //
 // This function satisfies the go-flags Commander interface.
 func (c *cmdLegacyTest) Execute(args []string) error {
-	printf("Policy\n")
+	// Verify admin login credentials
+	admin := user{
+		Email:    c.Args.AdminEmail,
+		Password: c.Args.AdminPassword,
+	}
+	err := userLogin(admin)
+	if err != nil {
+		return fmt.Errorf("failed to login admin: %v", err)
+	}
+	lr, err := client.Me()
+	if err != nil {
+		return err
+	}
+	if !lr.IsAdmin {
+		return fmt.Errorf("provided user is not an admin")
+	}
+	admin.Username = lr.Username
+
+	// Verify paywall is disabled
+	policyWWW, err := client.Policy()
+	if err != nil {
+		return err
+	}
+	if policyWWW.PaywallEnabled {
+		return fmt.Errorf("paywall is not disabled")
+	}
+
+	// Seed the backend with proposals
+	proposalCount := 10
+	tokens := make([]string, 0, proposalCount)
+	for i := 0; i < proposalCount; i++ {
+		s := fmt.Sprintf("Creating proposal %v/%v", i+1, proposalCount)
+		printInPlace(s)
+
+		r, err := proposalPublic(admin, admin, false)
+		if err != nil {
+			return err
+		}
+		tokens = append(tokens, r.CensorshipRecord.Token)
+	}
+	fmt.Printf("\n")
+
+	// Start the voting period on all proposals
+	for i, v := range tokens {
+		s := fmt.Sprintf("Starting voting period %v/%v", i+1, proposalCount)
+		printInPlace(s)
+
+		err = voteAuthorize(admin, v)
+		if err != nil {
+			return err
+		}
+		err = voteStart(admin, v, 1000, 1, 50)
+		if err != nil {
+			return err
+		}
+	}
+	fmt.Printf("\n")
+
+	fmt.Printf("Policy\n")
 	pr, err := client.Policy()
 	if err != nil {
 		return err
 	}
 	printJSON(pr)
 
-	printf("Token inventory\n")
+	fmt.Printf("Token inventory\n")
 	tir, err := client.TokenInventory()
 	if err != nil {
 		return err
 	}
 	printJSON(tir)
 
-	printf("All vetted\n")
+	fmt.Printf("All vetted\n")
 	avr, err := client.GetAllVetted(&www.GetAllVetted{})
 	if err != nil {
 		return err
 	}
 	printJSON(avr)
 
-	if len(tir.Pre) == 0 {
-		return fmt.Errorf("no proposals found; cannot get proposal details")
-	}
-	token := tir.Pre[0]
-
-	printf("Proposal details %v\n", token)
+	token := tokens[0]
+	fmt.Printf("Proposal details %v\n", token)
 	pdr, err := client.ProposalDetails(token, &www.ProposalsDetails{})
 	if err != nil {
 		return err
 	}
 	printJSON(pdr)
 
-	printf("Batch proposals %v\n", token)
+	fmt.Printf("Batch proposals\n")
 	bp := www.BatchProposals{
-		Tokens: []string{token},
+		Tokens: tokens,
 	}
 	bpr, err := client.BatchProposals(&bp)
 	if err != nil {
 		return err
 	}
+	if len(bpr.Proposals) != proposalCount {
+		return fmt.Errorf("got %v proposals, want %v",
+			len(bpr.Proposals), proposalCount)
+	}
 	printJSON(bpr)
 
-	printf("All vote status\n")
+	fmt.Printf("All vote status\n")
 	avsr, err := client.GetAllVoteStatus()
 	if err != nil {
 		return err
@@ -72,20 +136,20 @@ func (c *cmdLegacyTest) Execute(args []string) error {
 	}
 	token = tir.Approved[0]
 
-	printf("Vote status %v\n", token)
+	fmt.Printf("Vote status %v\n", token)
 	vsr, err := client.VoteStatus(token)
 	if err != nil {
 		return err
 	}
 	printJSON(vsr)
 
-	printf("Vote results %v\n", token)
+	fmt.Printf("Vote results %v\n", token)
 	vrr, err := client.VoteResults(token)
 	if err != nil {
 		return err
 	}
 	vrr.StartVoteReply.EligibleTickets = []string{
-		fmt.Sprintf("%v ticket hashes removed for readability",
+		fmt.Sprint("%v ticket hashes removed for readability",
 			len(vrr.StartVoteReply.EligibleTickets)),
 	}
 	printJSON(vrr)

politeiawww/proposals.go

@@ -32,47 +32,61 @@ import (
 )
 
 func (p *politeiawww) proposals(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]www.ProposalRecord, error) {
-	records, err := p.politeiad.Records(ctx, reqs)
-	if err != nil {
-		return nil, err
-	}
-
-	proposals := make(map[string]www.ProposalRecord, len(records))
-	for k, v := range records {
-		// Legacy www routes are only for vetted records
-		if v.State == pdv2.RecordStateUnvetted {
-			continue
+	// Break the requests up so that they do not exceed the politeiad
+	// records page size.
+	var startIdx int
+	proposals := make(map[string]www.ProposalRecord, len(reqs))
+	for startIdx < len(reqs) {
+		// Setup a page of requests
+		endIdx := startIdx + int(pdv2.RecordsPageSize)
+		if endIdx > len(reqs) {
+			endIdx = len(reqs)
 		}
 
-		// Convert to a proposal
-		pr, err := convertRecordToProposal(v)
+		records, err := p.politeiad.Records(ctx, reqs[startIdx:endIdx])
 		if err != nil {
 			return nil, err
 		}
 
-		// Get submissions list if this is an RFP
-		if pr.LinkBy != 0 {
-			subs, err := p.politeiad.TicketVoteSubmissions(ctx,
-				pr.CensorshipRecord.Token)
+		for k, v := range records {
+			// Legacy www routes are only for vetted records
+			if v.State == pdv2.RecordStateUnvetted {
+				continue
+			}
+
+			// Convert to a proposal
+			pr, err := convertRecordToProposal(v)
 			if err != nil {
 				return nil, err
 			}
-			pr.LinkedFrom = subs
-		}
 
-		// Fill in user data
-		userID := userIDFromMetadataStreams(v.Metadata)
-		uid, err := uuid.Parse(userID)
-		if err != nil {
-			return nil, err
-		}
-		u, err := p.db.UserGetById(uid)
-		if err != nil {
-			return nil, err
+			// Get submissions list if this is an RFP
+			if pr.LinkBy != 0 {
+				subs, err := p.politeiad.TicketVoteSubmissions(ctx,
+					pr.CensorshipRecord.Token)
+				if err != nil {
+					return nil, err
+				}
+				pr.LinkedFrom = subs
+			}
+
+			// Fill in user data
+			userID := userIDFromMetadataStreams(v.Metadata)
+			uid, err := uuid.Parse(userID)
+			if err != nil {
+				return nil, err
+			}
+			u, err := p.db.UserGetById(uid)
+			if err != nil {
+				return nil, err
+			}
+			pr.Username = u.Username
+
+			proposals[k] = *pr
 		}
-		pr.Username = u.Username
 
-		proposals[k] = *pr
+		// Update the index
+		startIdx = endIdx
 	}
 
 	return proposals, nil
@@ -176,9 +190,8 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted
 		return nil, err
 	}
 
-	// Get the records without any files
+	// Get the proposals without any files
 	reqs := make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize)
-	proposals := make([]www.ProposalRecord, 0, len(tokens))
 	for _, v := range tokens {
 		reqs = append(reqs, pdv2.RecordRequest{
 			Token: v,
@@ -187,26 +200,20 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted
 				tkplugin.FileNameVoteMetadata,
 			},
 		})
+	}
+	props, err := p.proposals(ctx, reqs)
+	if err != nil {
+		return nil, err
+	}
 
-		// The records request must be broken up because the records
-		// page size is much smaller than the inventory page size.
-		if len(reqs) == int(pdv2.RecordsPageSize) {
-			// Get this batch of proposals
-			props, err := p.proposals(ctx, reqs)
-			if err != nil {
-				return nil, err
-			}
-			for _, v := range reqs {
-				pr, ok := props[v.Token]
-				if !ok {
-					continue
-				}
-				proposals = append(proposals, pr)
-			}
-
-			// Reset requesets
-			reqs = make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize)
+	// Covert proposal map to an slice
+	proposals := make([]www.ProposalRecord, 0, len(props))
+	for _, v := range tokens {
+		pr, ok := props[v]
+		if !ok {
+			continue
 		}
+		proposals = append(proposals, pr)
 	}
 
 	return &www.GetAllVettedReply{
@@ -261,6 +268,7 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro
 		}
 	}
 
+	// Get the proposals batch
 	reqs := make([]pdv2.RecordRequest, 0, len(bp.Tokens))
 	for _, v := range bp.Tokens {
 		reqs = append(reqs, pdv2.RecordRequest{
@@ -275,13 +283,19 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro
 	if err != nil {
 		return nil, err
 	}
-	prs := make([]www.ProposalRecord, 0, len(props))
-	for _, v := range props {
-		prs = append(prs, v)
+
+	// Return the proposals in the same order they were requests in.
+	proposals := make([]www.ProposalRecord, 0, len(props))
+	for _, v := range bp.Tokens {
+		pr, ok := props[v]
+		if !ok {
+			continue
+		}
+		proposals = append(proposals, pr)
 	}
 
 	return &www.BatchProposalsReply{
-		Proposals: prs,
+		Proposals: proposals,
 	}, nil
 }
 
@@ -438,40 +452,20 @@ func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteRep
 		}, nil
 	}
 
-	// Get the proposal details, without the actual proposal files, of
-	// all proposals that are currently being voted on. These requests
-	// must be paginated so that the politeiad maximum records page
-	// size is not exceeded.
+	// Get proposals
 	reqs := make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize)
-	props := make(map[string]www.ProposalRecord, len(started))
-	for i, v := range started {
+	for _, v := range started {
 		reqs = append(reqs, pdv2.RecordRequest{
 			Token: v,
 			Filenames: []string{
 				piplugin.FileNameProposalMetadata,
 				tkplugin.FileNameVoteMetadata,
 			},
 		})
-		switch {
-		case i == len(started)-1:
-			// This is the last index. We must send the proposals request
-			// even if it is not requesting a full page. Continue to the
-			// code below.
-		case len(reqs) < int(pdv2.RecordsPageSize):
-			// Page size is not exceeded yet. Keep adding requests.
-			continue
-		}
-
-		// Retrieve this page of records, save the results, and reset
-		// the requests.
-		ps, err := p.proposals(ctx, reqs)
-		if err != nil {
-			return nil, err
-		}
-		for token, prop := range ps {
-			props[token] = prop
-		}
-		reqs = make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize)
+	}
+	props, err := p.proposals(ctx, reqs)
+	if err != nil {
+		return nil, err
 	}
 
 	// Get vote details