Catchpoints: Enrich catchpoint generation and status with KV metadata

.github/workflows/codegen_verification.yml

@@ -25,4 +25,4 @@ jobs:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         run: |
           curl -X POST --data-urlencode "payload={\"text\": \"Codegen verification failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
-        if: ${{ failure() && (contains(github.base_ref, 'rel/nightly') || contains(github.base_ref, 'rel/beta') || contains(github.base_ref, 'rel/stable') || contains(github.base_ref, 'master')) }}
+        if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

catchup/catchpointService.go

@@ -53,11 +53,15 @@ type CatchpointCatchupStats struct {
 	TotalAccounts      uint64
 	ProcessedAccounts  uint64
 	VerifiedAccounts   uint64
+	TotalKVs           uint64
+	ProcessedKVs       uint64
+	VerifiedKVs        uint64
 	TotalBlocks        uint64
 	AcquiredBlocks     uint64
 	VerifiedBlocks     uint64
 	ProcessedBytes     uint64
 	TotalAccountHashes uint64
+	TotalKVHashes      uint64
 	StartTime          time.Time
 }
 
@@ -303,7 +307,7 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 		if err == nil {
 			cs.log.Infof("ledger downloaded in %d seconds", time.Since(start)/time.Second)
 			start = time.Now()
-			err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedAccounts)
+			err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedCounts)
 			if err == nil {
 				cs.log.Infof("built merkle trie in %d seconds", time.Since(start)/time.Second)
 				break
@@ -335,12 +339,17 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 	return nil
 }
 
-// updateVerifiedAccounts update the user's statistics for the given verified accounts
-func (cs *CatchpointCatchupService) updateVerifiedAccounts(addedTrieHashes uint64) {
+// updateVerifiedCounts update the user's statistics for the given verified hashes
+func (cs *CatchpointCatchupService) updateVerifiedCounts(accountCount, kvCount uint64) {
 	cs.statsMu.Lock()
 	defer cs.statsMu.Unlock()
+
 	if cs.stats.TotalAccountHashes > 0 {
-		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * addedTrieHashes / cs.stats.TotalAccountHashes
+		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * accountCount / cs.stats.TotalAccountHashes
+	}
+
+	if cs.stats.TotalKVs > 0 {
+		cs.stats.VerifiedKVs = kvCount
 	}
 }
 
@@ -756,6 +765,8 @@ func (cs *CatchpointCatchupService) updateLedgerFetcherProgress(fetcherStats *le
 	defer cs.statsMu.Unlock()
 	cs.stats.TotalAccounts = fetcherStats.TotalAccounts
 	cs.stats.ProcessedAccounts = fetcherStats.ProcessedAccounts
+	cs.stats.TotalKVs = fetcherStats.TotalKVs
+	cs.stats.ProcessedKVs = fetcherStats.ProcessedKVs
 	cs.stats.ProcessedBytes = fetcherStats.ProcessedBytes
 	cs.stats.TotalAccountHashes = fetcherStats.TotalAccountHashes
 }

cmd/catchpointdump/file.go

@@ -266,6 +266,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 			"Block Header Digest: %s",
 			"Catchpoint: %s",
 			"Total Accounts: %d",
+			"Total KVs: %d",
 			"Total Chunks: %d",
 		}
 		var headerValues = []interface{}{
@@ -275,6 +276,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 			fileHeader.BlockHeaderDigest.String(),
 			fileHeader.Catchpoint,
 			fileHeader.TotalAccounts,
+			fileHeader.TotalKVs,
 			fileHeader.TotalChunks,
 		}
 		// safety check

cmd/goal/messages.go

@@ -64,7 +64,7 @@ const (
 	infoNodeStatus                     = "Last committed block: %d\nTime since last block: %s\nSync Time: %s\nLast consensus protocol: %s\nNext consensus protocol: %s\nRound for next consensus protocol: %d\nNext consensus protocol supported: %v"
 	catchupStoppedOnUnsupported        = "Last supported block (%d) is committed. The next block consensus protocol is not supported. Catchup service is stopped."
 	infoNodeCatchpointCatchupStatus    = "Last committed block: %d\nSync Time: %s\nCatchpoint: %s"
-	infoNodeCatchpointCatchupAccounts  = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d\nCatchpoint accounts verified: %d"
+	infoNodeCatchpointCatchupAccounts  = "Catchpoint total accounts: %d\nCatchpoint accounts processed: %d\nCatchpoint accounts verified: %d\nCatchpoint total KVs: %d\nCatchpoint KVs processed: %d\nCatchpoint KVs verified: %d"
 	infoNodeCatchpointCatchupBlocks    = "Catchpoint total blocks: %d\nCatchpoint downloaded blocks: %d"
 	nodeLastCatchpoint                 = "Last Catchpoint: %s"
 	errorNodeCreationIPFailure         = "Parsing passed IP %v failed: need a valid IPv4 or IPv6 address with a specified port number"

cmd/goal/node.go

@@ -467,7 +467,8 @@ func makeStatusString(stat model.NodeStatusResponse) string {
 
 		if stat.CatchpointTotalAccounts != nil && (*stat.CatchpointTotalAccounts > 0) && stat.CatchpointProcessedAccounts != nil {
 			statusString = statusString + "\n" + fmt.Sprintf(infoNodeCatchpointCatchupAccounts, *stat.CatchpointTotalAccounts,
-				*stat.CatchpointProcessedAccounts, *stat.CatchpointVerifiedAccounts)
+				*stat.CatchpointProcessedAccounts, *stat.CatchpointVerifiedAccounts,
+				*stat.CatchpointTotalKvs, *stat.CatchpointProcessedKvs, *stat.CatchpointVerifiedKvs)
 		}
 		if stat.CatchpointAcquiredBlocks != nil && stat.CatchpointTotalBlocks != nil && (*stat.CatchpointAcquiredBlocks+*stat.CatchpointTotalBlocks > 0) {
 			statusString = statusString + "\n" + fmt.Sprintf(infoNodeCatchpointCatchupBlocks, *stat.CatchpointTotalBlocks,

components/mocks/mockCatchpointCatchupAccessor.go

@@ -58,7 +58,7 @@ func (m *MockCatchpointCatchupAccessor) ProcessStagingBalances(ctx context.Conte
 }
 
 // BuildMerkleTrie inserts the account hashes into the merkle trie
-func (m *MockCatchpointCatchupAccessor) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64)) (err error) {
+func (m *MockCatchpointCatchupAccessor) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error) {
 	return nil
 }
 

daemon/algod/api/algod.oas2.json

@@ -3523,6 +3523,18 @@
             "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup",
             "type": "integer"
           },
+          "catchpoint-total-kvs": {
+            "description": "The total number of key-values (KVs) included in the current catchpoint",
+            "type": "integer"
+          },
+          "catchpoint-processed-kvs": {
+            "description": "The number of key-values (KVs) from the current catchpoint that have been processed so far as part of the catchup",
+            "type": "integer"
+          },
+          "catchpoint-verified-kvs": {
+            "description": "The number of key-values (KVs) from the current catchpoint that have been verified so far as part of the catchup",
+            "type": "integer"
+          },
           "catchpoint-total-blocks": {
             "description": "The total number of blocks that are required to complete the current catchpoint catchup",
             "type": "integer"

daemon/algod/api/algod.oas3.yml

@@ -525,6 +525,10 @@
                   "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup",
                   "type": "integer"
                 },
+                "catchpoint-processed-kvs": {
+                  "description": "The number of key-values (KVs) from the current catchpoint that have been processed so far as part of the catchup",
+                  "type": "integer"
+                },
                 "catchpoint-total-accounts": {
                   "description": "The total number of accounts included in the current catchpoint",
                   "type": "integer"
@@ -533,10 +537,18 @@
                   "description": "The total number of blocks that are required to complete the current catchpoint catchup",
                   "type": "integer"
                 },
+                "catchpoint-total-kvs": {
+                  "description": "The total number of key-values (KVs) included in the current catchpoint",
+                  "type": "integer"
+                },
                 "catchpoint-verified-accounts": {
                   "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup",
                   "type": "integer"
                 },
+                "catchpoint-verified-kvs": {
+                  "description": "The number of key-values (KVs) from the current catchpoint that have been verified so far as part of the catchup",
+                  "type": "integer"
+                },
                 "catchup-time": {
                   "description": "CatchupTime in nanoseconds",
                   "type": "integer"
@@ -4101,6 +4113,10 @@
                       "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup",
                       "type": "integer"
                     },
+                    "catchpoint-processed-kvs": {
+                      "description": "The number of key-values (KVs) from the current catchpoint that have been processed so far as part of the catchup",
+                      "type": "integer"
+                    },
                     "catchpoint-total-accounts": {
                       "description": "The total number of accounts included in the current catchpoint",
                       "type": "integer"
@@ -4109,10 +4125,18 @@
                       "description": "The total number of blocks that are required to complete the current catchpoint catchup",
                       "type": "integer"
                     },
+                    "catchpoint-total-kvs": {
+                      "description": "The total number of key-values (KVs) included in the current catchpoint",
+                      "type": "integer"
+                    },
                     "catchpoint-verified-accounts": {
                       "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup",
                       "type": "integer"
                     },
+                    "catchpoint-verified-kvs": {
+                      "description": "The number of key-values (KVs) from the current catchpoint that have been verified so far as part of the catchup",
+                      "type": "integer"
+                    },
                     "catchup-time": {
                       "description": "CatchupTime in nanoseconds",
                       "type": "integer"
@@ -4232,6 +4256,10 @@
                       "description": "The number of accounts from the current catchpoint that have been processed so far as part of the catchup",
                       "type": "integer"
                     },
+                    "catchpoint-processed-kvs": {
+                      "description": "The number of key-values (KVs) from the current catchpoint that have been processed so far as part of the catchup",
+                      "type": "integer"
+                    },
                     "catchpoint-total-accounts": {
                       "description": "The total number of accounts included in the current catchpoint",
                       "type": "integer"
@@ -4240,10 +4268,18 @@
                       "description": "The total number of blocks that are required to complete the current catchpoint catchup",
                       "type": "integer"
                     },
+                    "catchpoint-total-kvs": {
+                      "description": "The total number of key-values (KVs) included in the current catchpoint",
+                      "type": "integer"
+                    },
                     "catchpoint-verified-accounts": {
                       "description": "The number of accounts from the current catchpoint that have been verified so far as part of the catchup",
                       "type": "integer"
                     },
+                    "catchpoint-verified-kvs": {
+                      "description": "The number of key-values (KVs) from the current catchpoint that have been verified so far as part of the catchup",
+                      "type": "integer"
+                    },
                     "catchup-time": {
                       "description": "CatchupTime in nanoseconds",
                       "type": "integer"