From 60d0c0907662672bef12a36b7475cf7c6dea8b10 Mon Sep 17 00:00:00 2001 From: Ignacio Corderi Date: Mon, 21 Nov 2022 18:28:37 -0300 Subject: [PATCH] ledger: move pieces of accountdb.go into a storage package (#4776) --- Makefile | 4 +- ledger/accountdb.go | 1709 +-------- ledger/accountdb_test.go | 1729 ++------- ledger/acctonline.go | 36 +- ledger/acctonline_test.go | 196 +- ledger/acctupdates.go | 111 +- ledger/acctupdates_test.go | 61 +- ledger/applications_test.go | 78 +- ledger/blockdb_test.go | 21 +- ledger/catchpointtracker.go | 22 +- ledger/catchpointtracker_test.go | 11 +- ledger/catchupaccessor_test.go | 13 +- ledger/lruaccts.go | 21 +- ledger/lruaccts_test.go | 89 +- ledger/lrukv.go | 23 +- ledger/lrukv_test.go | 51 +- ledger/lruonlineaccts.go | 21 +- ledger/lruonlineaccts_test.go | 71 +- ledger/lruresources.go | 31 +- ledger/lruresources_test.go | 83 +- ledger/msgp_gen.go | 4118 ++++++---------------- ledger/msgp_gen_test.go | 240 -- ledger/onlineaccountscache.go | 9 +- ledger/onlineaccountscache_test.go | 17 +- ledger/persistedaccts_list.go | 6 +- ledger/persistedaccts_list_test.go | 27 +- ledger/persistedonlineaccts_list.go | 6 +- ledger/persistedonlineaccts_list_test.go | 27 +- ledger/store/data.go | 760 ++++ ledger/store/data_test.go | 1254 +++++++ ledger/store/interface.go | 75 + ledger/store/msgp_gen.go | 1880 ++++++++++ ledger/store/msgp_gen_test.go | 255 ++ ledger/store/sql.go | 716 ++++ ledger/store/sql_test.go | 55 + ledger/store/testing/helpers.go | 43 + ledger/tracker.go | 9 +- 37 files changed, 7090 insertions(+), 6788 deletions(-) create mode 100644 ledger/store/data.go create mode 100644 ledger/store/data_test.go create mode 100644 ledger/store/interface.go create mode 100644 ledger/store/msgp_gen.go create mode 100644 ledger/store/msgp_gen_test.go create mode 100644 ledger/store/sql.go create mode 100644 ledger/store/sql_test.go create mode 100644 ledger/store/testing/helpers.go diff --git a/Makefile b/Makefile index a7613fbf42..0296a6408d 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \ UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ )) ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... )) -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./stateproof ./data/account ./daemon/algod/api/spec/v2 +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./ledger/store ./stateproof ./data/account ./daemon/algod/api/spec/v2 default: build @@ -99,7 +99,7 @@ fix: build $(GOPATH1)/bin/algofix */ lint: deps - $(GOPATH1)/bin/golangci-lint run -c .golangci.yml + $(GOPATH1)/bin/golangci-lint run -c .golangci.yml check_shell: find . -type f -name "*.sh" -exec shellcheck {} + diff --git a/ledger/accountdb.go b/ledger/accountdb.go index 9b8d418287..5ba566bfa2 100644 --- a/ledger/accountdb.go +++ b/ledger/accountdb.go @@ -39,29 +39,12 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/db" ) -// accountsDbQueries is used to cache a prepared SQL statement to look up -// the state of a single account. -type accountsDbQueries struct { - listCreatablesStmt *sql.Stmt - lookupStmt *sql.Stmt - lookupResourcesStmt *sql.Stmt - lookupAllResourcesStmt *sql.Stmt - lookupKvPairStmt *sql.Stmt - lookupKeysByRangeStmt *sql.Stmt - lookupCreatorStmt *sql.Stmt -} - -type onlineAccountsDbQueries struct { - lookupOnlineStmt *sql.Stmt - lookupOnlineHistoryStmt *sql.Stmt - lookupOnlineTotalsStmt *sql.Stmt -} - var accountsSchema = []string{ `CREATE TABLE IF NOT EXISTS acctrounds ( id string primary key, @@ -195,96 +178,10 @@ var accountsResetExprs = []string{ // and their descriptions. var accountDBVersion = int32(9) -// persistedAccountData is used for representing a single account stored on the disk. In addition to the -// basics.AccountData, it also stores complete referencing information used to maintain the base accounts -// list. -type persistedAccountData struct { - // The address of the account. In contrasts to maps, having this value explicitly here allows us to use this - // data structure in queues directly, without "attaching" the address as the address as the map key. - addr basics.Address - // The underlaying account data - accountData baseAccountData - // The rowid, when available. If the entry was loaded from the disk, then we have the rowid for it. Entries - // that doesn't have rowid ( hence, rowid == 0 ) represent either deleted accounts or non-existing accounts. - rowid int64 - // the round number that is associated with the accountData. This field is needed so that we can maintain a correct - // lruAccounts cache. We use it to ensure that the entries on the lruAccounts.accountsList are the latest ones. - // this becomes an issue since while we attempt to write an update to disk, we might be reading an entry and placing - // it on the lruAccounts.pendingAccounts; The commitRound doesn't attempt to flush the pending accounts, but rather - // just write the latest ( which is correct ) to the lruAccounts.accountsList. later on, during on newBlockImpl, we - // want to ensure that the "real" written value isn't being overridden by the value from the pending accounts. - round basics.Round -} - -type persistedOnlineAccountData struct { - addr basics.Address - accountData baseOnlineAccountData - rowid int64 - // the round number that is associated with the baseOnlineAccountData. This field is the corresponding one to the round field - // in persistedAccountData, and serves the same purpose. This value comes from account rounds table and correspond to - // the last trackers db commit round. - round basics.Round - // the round number that the online account is for, i.e. account state change round. - updRound basics.Round -} - -//msgp:ignore persistedResourcesData -type persistedResourcesData struct { - // addrid is the rowid of the account address that holds this resource. - // it is used in update/delete operations so must be filled for existing records. - // resolution is a multi stage process: - // - baseResources cache might have valid entries - // - baseAccount cache might have an entry for the address with rowid set - // - when loading non-cached resources in resourcesLoadOld - // - when creating new accounts in accountsNewRound - addrid int64 - // creatable index - aidx basics.CreatableIndex - // actual resource data - data resourcesData - // the round number that is associated with the resourcesData. This field is the corresponding one to the round field - // in persistedAccountData, and serves the same purpose. - round basics.Round -} - -func (prd *persistedResourcesData) AccountResource() ledgercore.AccountResource { - var ret ledgercore.AccountResource - if prd.data.IsAsset() { - if prd.data.IsHolding() { - holding := prd.data.GetAssetHolding() - ret.AssetHolding = &holding - } - if prd.data.IsOwning() { - assetParams := prd.data.GetAssetParams() - ret.AssetParams = &assetParams - } - } - if prd.data.IsApp() { - if prd.data.IsHolding() { - localState := prd.data.GetAppLocalState() - ret.AppLocalState = &localState - } - if prd.data.IsOwning() { - appParams := prd.data.GetAppParams() - ret.AppParams = &appParams - } - } - return ret -} - -//msgp:ignore persistedKVData -type persistedKVData struct { - // kv value - value []byte - // the round number that is associated with the kv value. This field is the corresponding one to the round field - // in persistedAccountData, and serves the same purpose. - round basics.Round -} - // resourceDelta is used as part of the compactResourcesDeltas to describe a change to a single resource. type resourceDelta struct { - oldResource persistedResourcesData - newResource resourcesData + oldResource store.PersistedResourcesData + newResource store.ResourcesData nAcctDeltas int address basics.Address } @@ -302,8 +199,8 @@ type compactResourcesDeltas struct { } type accountDelta struct { - oldAcct persistedAccountData - newAcct baseAccountData + oldAcct store.PersistedAccountData + newAcct store.BaseAccountData nAcctDeltas int address basics.Address } @@ -325,8 +222,8 @@ type compactAccountDeltas struct { // oldAcct represents the "old" state of the account in the DB, and compared against newAcct[0] // to determine if the acct became online or went offline. type onlineAccountDelta struct { - oldAcct persistedOnlineAccountData - newAcct []baseOnlineAccountData + oldAcct store.PersistedOnlineAccountData + newAcct []store.BaseOnlineAccountData nOnlineAcctDeltas int address basics.Address updRound []uint64 @@ -386,9 +283,9 @@ type normalizedAccountBalance struct { // The public key address to which the account belongs. address basics.Address // accountData contains the baseAccountData for that account. - accountData baseAccountData + accountData store.BaseAccountData // resources is a map, where the key is the creatable index, and the value is the resource data. - resources map[basics.CreatableIndex]resourcesData + resources map[basics.CreatableIndex]store.ResourcesData // encodedAccountData contains the baseAccountData encoded bytes that are going to be written to the accountbase table. encodedAccountData []byte // accountHashes contains a list of all the hashes that would need to be added to the merkle trie for that account. @@ -416,11 +313,11 @@ func prepareNormalizedBalancesV5(bals []encodedBalanceRecordV5, proto config.Con normalizedAccountBalances[i].normalizedBalance = accountDataV5.NormalizedOnlineBalance(proto) type resourcesRow struct { aidx basics.CreatableIndex - resourcesData + store.ResourcesData } var resources []resourcesRow - addResourceRow := func(_ context.Context, _ int64, aidx basics.CreatableIndex, rd *resourcesData) error { - resources = append(resources, resourcesRow{aidx: aidx, resourcesData: *rd}) + addResourceRow := func(_ context.Context, _ int64, aidx basics.CreatableIndex, rd *store.ResourcesData) error { + resources = append(resources, resourcesRow{aidx: aidx, ResourcesData: *rd}) return nil } if err = accountDataResources(context.Background(), &accountDataV5, 0, addResourceRow); err != nil { @@ -429,12 +326,12 @@ func prepareNormalizedBalancesV5(bals []encodedBalanceRecordV5, proto config.Con normalizedAccountBalances[i].accountHashes = make([][]byte, 1) normalizedAccountBalances[i].accountHashes[0] = accountHashBuilder(balance.Address, accountDataV5, balance.AccountData) if len(resources) > 0 { - normalizedAccountBalances[i].resources = make(map[basics.CreatableIndex]resourcesData, len(resources)) + normalizedAccountBalances[i].resources = make(map[basics.CreatableIndex]store.ResourcesData, len(resources)) normalizedAccountBalances[i].encodedResources = make(map[basics.CreatableIndex][]byte, len(resources)) } for _, resource := range resources { - normalizedAccountBalances[i].resources[resource.aidx] = resource.resourcesData - normalizedAccountBalances[i].encodedResources[resource.aidx] = protocol.Encode(&resource.resourcesData) + normalizedAccountBalances[i].resources[resource.aidx] = resource.ResourcesData + normalizedAccountBalances[i].encodedResources[resource.aidx] = protocol.Encode(&resource.ResourcesData) } normalizedAccountBalances[i].encodedAccountData = protocol.Encode(&normalizedAccountBalances[i].accountData) } @@ -470,10 +367,10 @@ func prepareNormalizedBalancesV6(bals []encodedBalanceRecordV6, proto config.Con curHashIdx++ } if len(balance.Resources) > 0 { - normalizedAccountBalances[i].resources = make(map[basics.CreatableIndex]resourcesData, len(balance.Resources)) + normalizedAccountBalances[i].resources = make(map[basics.CreatableIndex]store.ResourcesData, len(balance.Resources)) normalizedAccountBalances[i].encodedResources = make(map[basics.CreatableIndex][]byte, len(balance.Resources)) for cidx, res := range balance.Resources { - var resData resourcesData + var resData store.ResourcesData err = protocol.Decode(res, &resData) if err != nil { return nil, err @@ -534,21 +431,21 @@ func makeCompactResourceDeltas(accountDeltas []ledgercore.AccountDeltas, baseRou newEntry := resourceDelta{ nAcctDeltas: 1, address: res.Addr, - newResource: makeResourcesData(deltaRound * updateRoundMultiplier), + newResource: store.MakeResourcesData(deltaRound * updateRoundMultiplier), } newEntry.newResource.SetAssetData(res.Params, res.Holding) // baseResources caches deleted entries, and they have addrid = 0 // need to handle this and prevent such entries to be treated as fully resolved baseResourceData, has := baseResources.read(res.Addr, basics.CreatableIndex(res.Aidx)) - existingAcctCacheEntry := has && baseResourceData.addrid != 0 + existingAcctCacheEntry := has && baseResourceData.Addrid != 0 if existingAcctCacheEntry { newEntry.oldResource = baseResourceData outResourcesDeltas.insert(newEntry) } else { if pad, has := baseAccounts.read(res.Addr); has { - newEntry.oldResource = persistedResourcesData{addrid: pad.rowid} + newEntry.oldResource = store.PersistedResourcesData{Addrid: pad.Rowid} } - newEntry.oldResource.aidx = basics.CreatableIndex(res.Aidx) + newEntry.oldResource.Aidx = basics.CreatableIndex(res.Aidx) outResourcesDeltas.insertMissing(newEntry) } } @@ -572,19 +469,19 @@ func makeCompactResourceDeltas(accountDeltas []ledgercore.AccountDeltas, baseRou newEntry := resourceDelta{ nAcctDeltas: 1, address: res.Addr, - newResource: makeResourcesData(deltaRound * updateRoundMultiplier), + newResource: store.MakeResourcesData(deltaRound * updateRoundMultiplier), } newEntry.newResource.SetAppData(res.Params, res.State) baseResourceData, has := baseResources.read(res.Addr, basics.CreatableIndex(res.Aidx)) - existingAcctCacheEntry := has && baseResourceData.addrid != 0 + existingAcctCacheEntry := has && baseResourceData.Addrid != 0 if existingAcctCacheEntry { newEntry.oldResource = baseResourceData outResourcesDeltas.insert(newEntry) } else { if pad, has := baseAccounts.read(res.Addr); has { - newEntry.oldResource = persistedResourcesData{addrid: pad.rowid} + newEntry.oldResource = store.PersistedResourcesData{Addrid: pad.Rowid} } - newEntry.oldResource.aidx = basics.CreatableIndex(res.Aidx) + newEntry.oldResource.Aidx = basics.CreatableIndex(res.Aidx) outResourcesDeltas.insertMissing(newEntry) } } @@ -622,9 +519,9 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx *sql.Tx, knownAddresses map for _, missIdx := range a.misses { delta := a.deltas[missIdx] addr := delta.address - aidx = delta.oldResource.aidx - if delta.oldResource.addrid != 0 { - addrid = delta.oldResource.addrid + aidx = delta.oldResource.Aidx + if delta.oldResource.Addrid != 0 { + addrid = delta.oldResource.Addrid } else if addrid, ok = knownAddresses[addr]; !ok { err = addrRowidStmt.QueryRow(addr[:]).Scan(&addrid) if err != nil { @@ -644,8 +541,8 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx *sql.Tx, knownAddresses map switch err { case nil: if len(resDataBuf) > 0 { - persistedResData := persistedResourcesData{addrid: addrid, aidx: aidx} - err = protocol.Decode(resDataBuf, &persistedResData.data) + persistedResData := store.PersistedResourcesData{Addrid: addrid, Aidx: aidx} + err = protocol.Decode(resDataBuf, &persistedResData.Data) if err != nil { return err } @@ -656,7 +553,7 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx *sql.Tx, knownAddresses map } case sql.ErrNoRows: // we don't have that account, just return an empty record. - a.updateOld(missIdx, persistedResourcesData{addrid: addrid, aidx: aidx}) + a.updateOld(missIdx, store.PersistedResourcesData{Addrid: addrid, Aidx: aidx}) err = nil default: // unexpected error - let the caller know that we couldn't complete the operation. @@ -696,7 +593,7 @@ func (a *compactResourcesDeltas) insert(delta resourceDelta) int { if a.cache == nil { a.cache = make(map[accountCreatable]int) } - a.cache[accountCreatable{address: delta.address, index: delta.oldResource.aidx}] = last + a.cache[accountCreatable{address: delta.address, index: delta.oldResource.Aidx}] = last return last } @@ -705,7 +602,7 @@ func (a *compactResourcesDeltas) insertMissing(delta resourceDelta) { } // updateOld updates existing or inserts a new partial entry with only old field filled -func (a *compactResourcesDeltas) updateOld(idx int, old persistedResourcesData) { +func (a *compactResourcesDeltas) updateOld(idx int, old store.PersistedResourcesData) { a.deltas[idx].oldResource = old } @@ -749,7 +646,7 @@ func makeCompactAccountDeltas(accountDeltas []ledgercore.AccountDeltas, baseRoun // it's a new entry. newEntry := accountDelta{ nAcctDeltas: 1, - newAcct: baseAccountData{ + newAcct: store.BaseAccountData{ UpdateRound: deltaRound * updateRoundMultiplier, }, address: addr, @@ -790,19 +687,19 @@ func (a *compactAccountDeltas) accountsLoadOld(tx *sql.Tx) (err error) { switch err { case nil: if len(acctDataBuf) > 0 { - persistedAcctData := &persistedAccountData{addr: addr, rowid: rowid.Int64} - err = protocol.Decode(acctDataBuf, &persistedAcctData.accountData) + persistedAcctData := &store.PersistedAccountData{Addr: addr, Rowid: rowid.Int64} + err = protocol.Decode(acctDataBuf, &persistedAcctData.AccountData) if err != nil { return err } a.updateOld(idx, *persistedAcctData) } else { // to retain backward compatibility, we will treat this condition as if we don't have the account. - a.updateOld(idx, persistedAccountData{addr: addr, rowid: rowid.Int64}) + a.updateOld(idx, store.PersistedAccountData{Addr: addr, Rowid: rowid.Int64}) } case sql.ErrNoRows: // we don't have that account, just return an empty record. - a.updateOld(idx, persistedAccountData{addr: addr}) + a.updateOld(idx, store.PersistedAccountData{Addr: addr}) err = nil default: // unexpected error - let the caller know that we couldn't complete the operation. @@ -852,12 +749,12 @@ func (a *compactAccountDeltas) insertMissing(delta accountDelta) { } // updateOld updates existing or inserts a new partial entry with only old field filled -func (a *compactAccountDeltas) updateOld(idx int, old persistedAccountData) { +func (a *compactAccountDeltas) updateOld(idx int, old store.PersistedAccountData) { a.deltas[idx].oldAcct = old } func (c *onlineAccountDelta) append(acctDelta ledgercore.AccountData, deltaRound basics.Round) { - var baseEntry baseOnlineAccountData + var baseEntry store.BaseOnlineAccountData baseEntry.SetCoreAccountData(&acctDelta) c.newAcct = append(c.newAcct, baseEntry) c.updRound = append(c.updRound, uint64(deltaRound)) @@ -933,19 +830,19 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx *sql.Tx) (err error) { switch err { case nil: if len(acctDataBuf) > 0 { - persistedAcctData := &persistedOnlineAccountData{addr: addr, rowid: rowid.Int64} - err = protocol.Decode(acctDataBuf, &persistedAcctData.accountData) + persistedAcctData := &store.PersistedOnlineAccountData{Addr: addr, Rowid: rowid.Int64} + err = protocol.Decode(acctDataBuf, &persistedAcctData.AccountData) if err != nil { return err } a.updateOld(idx, *persistedAcctData) } else { // empty data means offline account - a.updateOld(idx, persistedOnlineAccountData{addr: addr, rowid: rowid.Int64}) + a.updateOld(idx, store.PersistedOnlineAccountData{Addr: addr, Rowid: rowid.Int64}) } case sql.ErrNoRows: // we don't have that account, just return an empty record. - a.updateOld(idx, persistedOnlineAccountData{addr: addr}) + a.updateOld(idx, store.PersistedOnlineAccountData{Addr: addr}) err = nil default: // unexpected error - let the caller know that we couldn't complete the operation. @@ -995,7 +892,7 @@ func (a *compactOnlineAccountDeltas) insertMissing(delta onlineAccountDelta) { } // updateOld updates existing or inserts a new partial entry with only old field filled -func (a *compactOnlineAccountDeltas) updateOld(idx int, old persistedOnlineAccountData) { +func (a *compactOnlineAccountDeltas) updateOld(idx int, old store.PersistedOnlineAccountData) { a.deltas[idx].oldAcct = old } @@ -1460,597 +1357,14 @@ func accountsCreateUnfinishedCatchpointsTable(ctx context.Context, e db.Executab return err } -type baseVotingData struct { - _struct struct{} `codec:",omitempty,omitemptyarray"` - - VoteID crypto.OneTimeSignatureVerifier `codec:"A"` - SelectionID crypto.VRFVerifier `codec:"B"` - VoteFirstValid basics.Round `codec:"C"` - VoteLastValid basics.Round `codec:"D"` - VoteKeyDilution uint64 `codec:"E"` - StateProofID merklesignature.Commitment `codec:"F"` -} - -type baseOnlineAccountData struct { - _struct struct{} `codec:",omitempty,omitemptyarray"` - - baseVotingData - - MicroAlgos basics.MicroAlgos `codec:"Y"` - RewardsBase uint64 `codec:"Z"` -} - -type baseAccountData struct { - _struct struct{} `codec:",omitempty,omitemptyarray"` - - Status basics.Status `codec:"a"` - MicroAlgos basics.MicroAlgos `codec:"b"` - RewardsBase uint64 `codec:"c"` - RewardedMicroAlgos basics.MicroAlgos `codec:"d"` - AuthAddr basics.Address `codec:"e"` - TotalAppSchemaNumUint uint64 `codec:"f"` - TotalAppSchemaNumByteSlice uint64 `codec:"g"` - TotalExtraAppPages uint32 `codec:"h"` - TotalAssetParams uint64 `codec:"i"` - TotalAssets uint64 `codec:"j"` - TotalAppParams uint64 `codec:"k"` - TotalAppLocalStates uint64 `codec:"l"` - TotalBoxes uint64 `codec:"m"` - TotalBoxBytes uint64 `codec:"n"` - - baseVotingData - - // UpdateRound is the round that modified this account data last. Since we want all the nodes to have the exact same - // value for this field, we'll be setting the value of this field to zero *before* the EnableAccountDataResourceSeparation - // consensus parameter is being set. Once the above consensus takes place, this field would be populated with the - // correct round number. - UpdateRound uint64 `codec:"z"` -} - -// IsEmpty return true if any of the fields other then the UpdateRound are non-zero. -func (ba *baseAccountData) IsEmpty() bool { - return ba.Status == 0 && - ba.MicroAlgos.Raw == 0 && - ba.RewardsBase == 0 && - ba.RewardedMicroAlgos.Raw == 0 && - ba.AuthAddr.IsZero() && - ba.TotalAppSchemaNumUint == 0 && - ba.TotalAppSchemaNumByteSlice == 0 && - ba.TotalExtraAppPages == 0 && - ba.TotalAssetParams == 0 && - ba.TotalAssets == 0 && - ba.TotalAppParams == 0 && - ba.TotalAppLocalStates == 0 && - ba.TotalBoxes == 0 && - ba.TotalBoxBytes == 0 && - ba.baseVotingData.IsEmpty() -} - -func (ba *baseAccountData) NormalizedOnlineBalance(proto config.ConsensusParams) uint64 { - return basics.NormalizedOnlineAccountBalance(ba.Status, ba.RewardsBase, ba.MicroAlgos, proto) -} - -func (ba *baseAccountData) SetCoreAccountData(ad *ledgercore.AccountData) { - ba.Status = ad.Status - ba.MicroAlgos = ad.MicroAlgos - ba.RewardsBase = ad.RewardsBase - ba.RewardedMicroAlgos = ad.RewardedMicroAlgos - ba.AuthAddr = ad.AuthAddr - ba.TotalAppSchemaNumUint = ad.TotalAppSchema.NumUint - ba.TotalAppSchemaNumByteSlice = ad.TotalAppSchema.NumByteSlice - ba.TotalExtraAppPages = ad.TotalExtraAppPages - ba.TotalAssetParams = ad.TotalAssetParams - ba.TotalAssets = ad.TotalAssets - ba.TotalAppParams = ad.TotalAppParams - ba.TotalAppLocalStates = ad.TotalAppLocalStates - ba.TotalBoxes = ad.TotalBoxes - ba.TotalBoxBytes = ad.TotalBoxBytes - - ba.baseVotingData.SetCoreAccountData(ad) -} - -func (ba *baseAccountData) SetAccountData(ad *basics.AccountData) { - ba.Status = ad.Status - ba.MicroAlgos = ad.MicroAlgos - ba.RewardsBase = ad.RewardsBase - ba.RewardedMicroAlgos = ad.RewardedMicroAlgos - ba.AuthAddr = ad.AuthAddr - ba.TotalAppSchemaNumUint = ad.TotalAppSchema.NumUint - ba.TotalAppSchemaNumByteSlice = ad.TotalAppSchema.NumByteSlice - ba.TotalExtraAppPages = ad.TotalExtraAppPages - ba.TotalAssetParams = uint64(len(ad.AssetParams)) - ba.TotalAssets = uint64(len(ad.Assets)) - ba.TotalAppParams = uint64(len(ad.AppParams)) - ba.TotalAppLocalStates = uint64(len(ad.AppLocalStates)) - ba.TotalBoxes = ad.TotalBoxes - ba.TotalBoxBytes = ad.TotalBoxBytes - - ba.baseVotingData.VoteID = ad.VoteID - ba.baseVotingData.SelectionID = ad.SelectionID - ba.baseVotingData.StateProofID = ad.StateProofID - ba.baseVotingData.VoteFirstValid = ad.VoteFirstValid - ba.baseVotingData.VoteLastValid = ad.VoteLastValid - ba.baseVotingData.VoteKeyDilution = ad.VoteKeyDilution -} - -func (ba *baseAccountData) GetLedgerCoreAccountData() ledgercore.AccountData { - return ledgercore.AccountData{ - AccountBaseData: ba.GetLedgerCoreAccountBaseData(), - VotingData: ba.GetLedgerCoreVotingData(), - } -} - -func (ba *baseAccountData) GetLedgerCoreAccountBaseData() ledgercore.AccountBaseData { - return ledgercore.AccountBaseData{ - Status: ba.Status, - MicroAlgos: ba.MicroAlgos, - RewardsBase: ba.RewardsBase, - RewardedMicroAlgos: ba.RewardedMicroAlgos, - AuthAddr: ba.AuthAddr, - TotalAppSchema: basics.StateSchema{ - NumUint: ba.TotalAppSchemaNumUint, - NumByteSlice: ba.TotalAppSchemaNumByteSlice, - }, - TotalExtraAppPages: ba.TotalExtraAppPages, - TotalAppParams: ba.TotalAppParams, - TotalAppLocalStates: ba.TotalAppLocalStates, - TotalAssetParams: ba.TotalAssetParams, - TotalAssets: ba.TotalAssets, - TotalBoxes: ba.TotalBoxes, - TotalBoxBytes: ba.TotalBoxBytes, - } -} - -func (ba *baseAccountData) GetLedgerCoreVotingData() ledgercore.VotingData { - return ledgercore.VotingData{ - VoteID: ba.VoteID, - SelectionID: ba.SelectionID, - StateProofID: ba.StateProofID, - VoteFirstValid: ba.VoteFirstValid, - VoteLastValid: ba.VoteLastValid, - VoteKeyDilution: ba.VoteKeyDilution, - } -} - -func (ba *baseAccountData) GetAccountData() basics.AccountData { - return basics.AccountData{ - Status: ba.Status, - MicroAlgos: ba.MicroAlgos, - RewardsBase: ba.RewardsBase, - RewardedMicroAlgos: ba.RewardedMicroAlgos, - AuthAddr: ba.AuthAddr, - TotalAppSchema: basics.StateSchema{ - NumUint: ba.TotalAppSchemaNumUint, - NumByteSlice: ba.TotalAppSchemaNumByteSlice, - }, - TotalExtraAppPages: ba.TotalExtraAppPages, - TotalBoxes: ba.TotalBoxes, - TotalBoxBytes: ba.TotalBoxBytes, - - VoteID: ba.VoteID, - SelectionID: ba.SelectionID, - StateProofID: ba.StateProofID, - VoteFirstValid: ba.VoteFirstValid, - VoteLastValid: ba.VoteLastValid, - VoteKeyDilution: ba.VoteKeyDilution, - } -} - -// IsEmpty returns true if all of the fields are zero. -func (bv baseVotingData) IsEmpty() bool { - return bv == baseVotingData{} -} - -// SetCoreAccountData initializes baseVotingData from ledgercore.AccountData -func (bv *baseVotingData) SetCoreAccountData(ad *ledgercore.AccountData) { - bv.VoteID = ad.VoteID - bv.SelectionID = ad.SelectionID - bv.StateProofID = ad.StateProofID - bv.VoteFirstValid = ad.VoteFirstValid - bv.VoteLastValid = ad.VoteLastValid - bv.VoteKeyDilution = ad.VoteKeyDilution -} - -// IsVotingEmpty checks if voting data fields are empty -func (bo *baseOnlineAccountData) IsVotingEmpty() bool { - return bo.baseVotingData.IsEmpty() -} - -// IsEmpty return true if any of the fields are non-zero. -func (bo *baseOnlineAccountData) IsEmpty() bool { - return bo.IsVotingEmpty() && - bo.MicroAlgos.Raw == 0 && - bo.RewardsBase == 0 -} - -// GetOnlineAccount returns ledgercore.OnlineAccount for top online accounts / voters -// TODO: unify -func (bo *baseOnlineAccountData) GetOnlineAccount(addr basics.Address, normBalance uint64) ledgercore.OnlineAccount { - return ledgercore.OnlineAccount{ - Address: addr, - MicroAlgos: bo.MicroAlgos, - RewardsBase: bo.RewardsBase, - NormalizedOnlineBalance: normBalance, - VoteFirstValid: bo.VoteFirstValid, - VoteLastValid: bo.VoteLastValid, - StateProofID: bo.StateProofID, - } -} - -// GetOnlineAccountData returns basics.OnlineAccountData for lookup agreement -// TODO: unify with GetOnlineAccount/ledgercore.OnlineAccount -func (bo *baseOnlineAccountData) GetOnlineAccountData(proto config.ConsensusParams, rewardsLevel uint64) ledgercore.OnlineAccountData { - microAlgos, _, _ := basics.WithUpdatedRewards( - proto, basics.Online, bo.MicroAlgos, basics.MicroAlgos{}, bo.RewardsBase, rewardsLevel, - ) - - return ledgercore.OnlineAccountData{ - MicroAlgosWithRewards: microAlgos, - VotingData: ledgercore.VotingData{ - VoteID: bo.VoteID, - SelectionID: bo.SelectionID, - StateProofID: bo.StateProofID, - VoteFirstValid: bo.VoteFirstValid, - VoteLastValid: bo.VoteLastValid, - VoteKeyDilution: bo.VoteKeyDilution, - }, - } -} - -func (bo *baseOnlineAccountData) NormalizedOnlineBalance(proto config.ConsensusParams) uint64 { - return basics.NormalizedOnlineAccountBalance(basics.Online, bo.RewardsBase, bo.MicroAlgos, proto) -} - -func (bo *baseOnlineAccountData) SetCoreAccountData(ad *ledgercore.AccountData) { - bo.baseVotingData.SetCoreAccountData(ad) - - // MicroAlgos/RewardsBase are updated by the evaluator when accounts are touched - bo.MicroAlgos = ad.MicroAlgos - bo.RewardsBase = ad.RewardsBase -} - -type resourceFlags uint8 - -const ( - resourceFlagsHolding resourceFlags = 0 - resourceFlagsNotHolding resourceFlags = 1 - resourceFlagsOwnership resourceFlags = 2 - resourceFlagsEmptyAsset resourceFlags = 4 - resourceFlagsEmptyApp resourceFlags = 8 -) - -// -// Resource flags interpretation: -// -// resourceFlagsHolding - the resource contains the holding of asset/app. -// resourceFlagsNotHolding - the resource is completely empty. This state should not be persisted. -// resourceFlagsOwnership - the resource contains the asset parameter or application parameters. -// resourceFlagsEmptyAsset - this is an asset resource, and it is empty. -// resourceFlagsEmptyApp - this is an app resource, and it is empty. - -type resourcesData struct { - _struct struct{} `codec:",omitempty,omitemptyarray"` - - // asset parameters ( basics.AssetParams ) - Total uint64 `codec:"a"` - Decimals uint32 `codec:"b"` - DefaultFrozen bool `codec:"c"` - UnitName string `codec:"d"` - AssetName string `codec:"e"` - URL string `codec:"f"` - MetadataHash [32]byte `codec:"g"` - Manager basics.Address `codec:"h"` - Reserve basics.Address `codec:"i"` - Freeze basics.Address `codec:"j"` - Clawback basics.Address `codec:"k"` - - // asset holding ( basics.AssetHolding ) - Amount uint64 `codec:"l"` - Frozen bool `codec:"m"` - - // application local state ( basics.AppLocalState ) - SchemaNumUint uint64 `codec:"n"` - SchemaNumByteSlice uint64 `codec:"o"` - KeyValue basics.TealKeyValue `codec:"p"` - - // application global params ( basics.AppParams ) - ApprovalProgram []byte `codec:"q,allocbound=config.MaxAvailableAppProgramLen"` - ClearStateProgram []byte `codec:"r,allocbound=config.MaxAvailableAppProgramLen"` - GlobalState basics.TealKeyValue `codec:"s"` - LocalStateSchemaNumUint uint64 `codec:"t"` - LocalStateSchemaNumByteSlice uint64 `codec:"u"` - GlobalStateSchemaNumUint uint64 `codec:"v"` - GlobalStateSchemaNumByteSlice uint64 `codec:"w"` - ExtraProgramPages uint32 `codec:"x"` - - // ResourceFlags helps to identify which portions of this structure should be used; in particular, it - // helps to provide a marker - i.e. whether the account was, for instance, opted-in for the asset compared - // to just being the owner of the asset. A comparison against the empty structure doesn't work here - - // since both the holdings and the parameters are allowed to be all at their default values. - ResourceFlags resourceFlags `codec:"y"` - - // UpdateRound is the round that modified this resource last. Since we want all the nodes to have the exact same - // value for this field, we'll be setting the value of this field to zero *before* the EnableAccountDataResourceSeparation - // consensus parameter is being set. Once the above consensus takes place, this field would be populated with the - // correct round number. - UpdateRound uint64 `codec:"z"` -} - -// makeResourcesData returns a new empty instance of resourcesData. -// Using this constructor method is necessary because of the ResourceFlags field. -// An optional rnd args sets UpdateRound -func makeResourcesData(rnd uint64) resourcesData { - return resourcesData{ResourceFlags: resourceFlagsNotHolding, UpdateRound: rnd} -} - -func (rd *resourcesData) IsHolding() bool { - return (rd.ResourceFlags & resourceFlagsNotHolding) == resourceFlagsHolding -} - -func (rd *resourcesData) IsOwning() bool { - return (rd.ResourceFlags & resourceFlagsOwnership) == resourceFlagsOwnership -} - -func (rd *resourcesData) IsEmpty() bool { - return !rd.IsApp() && !rd.IsAsset() -} - -func (rd *resourcesData) IsEmptyAppFields() bool { - return rd.SchemaNumUint == 0 && - rd.SchemaNumByteSlice == 0 && - len(rd.KeyValue) == 0 && - len(rd.ApprovalProgram) == 0 && - len(rd.ClearStateProgram) == 0 && - len(rd.GlobalState) == 0 && - rd.LocalStateSchemaNumUint == 0 && - rd.LocalStateSchemaNumByteSlice == 0 && - rd.GlobalStateSchemaNumUint == 0 && - rd.GlobalStateSchemaNumByteSlice == 0 && - rd.ExtraProgramPages == 0 -} - -func (rd *resourcesData) IsApp() bool { - if (rd.ResourceFlags & resourceFlagsEmptyApp) == resourceFlagsEmptyApp { - return true - } - return !rd.IsEmptyAppFields() -} - -func (rd *resourcesData) IsEmptyAssetFields() bool { - return rd.Amount == 0 && - !rd.Frozen && - rd.Total == 0 && - rd.Decimals == 0 && - !rd.DefaultFrozen && - rd.UnitName == "" && - rd.AssetName == "" && - rd.URL == "" && - rd.MetadataHash == [32]byte{} && - rd.Manager.IsZero() && - rd.Reserve.IsZero() && - rd.Freeze.IsZero() && - rd.Clawback.IsZero() -} - -func (rd *resourcesData) IsAsset() bool { - if (rd.ResourceFlags & resourceFlagsEmptyAsset) == resourceFlagsEmptyAsset { - return true - } - return !rd.IsEmptyAssetFields() -} - -func (rd *resourcesData) ClearAssetParams() { - rd.Total = 0 - rd.Decimals = 0 - rd.DefaultFrozen = false - rd.UnitName = "" - rd.AssetName = "" - rd.URL = "" - rd.MetadataHash = basics.Address{} - rd.Manager = basics.Address{} - rd.Reserve = basics.Address{} - rd.Freeze = basics.Address{} - rd.Clawback = basics.Address{} - hadHolding := (rd.ResourceFlags & resourceFlagsNotHolding) == resourceFlagsHolding - rd.ResourceFlags -= rd.ResourceFlags & resourceFlagsOwnership - rd.ResourceFlags &= ^resourceFlagsEmptyAsset - if rd.IsEmptyAssetFields() && hadHolding { - rd.ResourceFlags |= resourceFlagsEmptyAsset - } -} - -func (rd *resourcesData) SetAssetParams(ap basics.AssetParams, haveHoldings bool) { - rd.Total = ap.Total - rd.Decimals = ap.Decimals - rd.DefaultFrozen = ap.DefaultFrozen - rd.UnitName = ap.UnitName - rd.AssetName = ap.AssetName - rd.URL = ap.URL - rd.MetadataHash = ap.MetadataHash - rd.Manager = ap.Manager - rd.Reserve = ap.Reserve - rd.Freeze = ap.Freeze - rd.Clawback = ap.Clawback - rd.ResourceFlags |= resourceFlagsOwnership - if !haveHoldings { - rd.ResourceFlags |= resourceFlagsNotHolding - } - rd.ResourceFlags &= ^resourceFlagsEmptyAsset - if rd.IsEmptyAssetFields() { - rd.ResourceFlags |= resourceFlagsEmptyAsset - } -} - -func (rd *resourcesData) GetAssetParams() basics.AssetParams { - ap := basics.AssetParams{ - Total: rd.Total, - Decimals: rd.Decimals, - DefaultFrozen: rd.DefaultFrozen, - UnitName: rd.UnitName, - AssetName: rd.AssetName, - URL: rd.URL, - MetadataHash: rd.MetadataHash, - Manager: rd.Manager, - Reserve: rd.Reserve, - Freeze: rd.Freeze, - Clawback: rd.Clawback, - } - return ap -} - -func (rd *resourcesData) ClearAssetHolding() { - rd.Amount = 0 - rd.Frozen = false - - rd.ResourceFlags |= resourceFlagsNotHolding - hadParams := (rd.ResourceFlags & resourceFlagsOwnership) == resourceFlagsOwnership - if hadParams && rd.IsEmptyAssetFields() { - rd.ResourceFlags |= resourceFlagsEmptyAsset - } else { - rd.ResourceFlags &= ^resourceFlagsEmptyAsset - } -} - -func (rd *resourcesData) SetAssetHolding(ah basics.AssetHolding) { - rd.Amount = ah.Amount - rd.Frozen = ah.Frozen - rd.ResourceFlags &= ^(resourceFlagsNotHolding + resourceFlagsEmptyAsset) - // resourceFlagsHolding is set implicitly since it is zero - if rd.IsEmptyAssetFields() { - rd.ResourceFlags |= resourceFlagsEmptyAsset - } -} - -func (rd *resourcesData) GetAssetHolding() basics.AssetHolding { - return basics.AssetHolding{ - Amount: rd.Amount, - Frozen: rd.Frozen, - } -} - -func (rd *resourcesData) ClearAppLocalState() { - rd.SchemaNumUint = 0 - rd.SchemaNumByteSlice = 0 - rd.KeyValue = nil - - rd.ResourceFlags |= resourceFlagsNotHolding - hadParams := (rd.ResourceFlags & resourceFlagsOwnership) == resourceFlagsOwnership - if hadParams && rd.IsEmptyAppFields() { - rd.ResourceFlags |= resourceFlagsEmptyApp - } else { - rd.ResourceFlags &= ^resourceFlagsEmptyApp - } -} - -func (rd *resourcesData) SetAppLocalState(als basics.AppLocalState) { - rd.SchemaNumUint = als.Schema.NumUint - rd.SchemaNumByteSlice = als.Schema.NumByteSlice - rd.KeyValue = als.KeyValue - rd.ResourceFlags &= ^(resourceFlagsEmptyApp + resourceFlagsNotHolding) - if rd.IsEmptyAppFields() { - rd.ResourceFlags |= resourceFlagsEmptyApp - } -} - -func (rd *resourcesData) GetAppLocalState() basics.AppLocalState { - return basics.AppLocalState{ - Schema: basics.StateSchema{ - NumUint: rd.SchemaNumUint, - NumByteSlice: rd.SchemaNumByteSlice, - }, - KeyValue: rd.KeyValue, - } -} - -func (rd *resourcesData) ClearAppParams() { - rd.ApprovalProgram = nil - rd.ClearStateProgram = nil - rd.GlobalState = nil - rd.LocalStateSchemaNumUint = 0 - rd.LocalStateSchemaNumByteSlice = 0 - rd.GlobalStateSchemaNumUint = 0 - rd.GlobalStateSchemaNumByteSlice = 0 - rd.ExtraProgramPages = 0 - hadHolding := (rd.ResourceFlags & resourceFlagsNotHolding) == resourceFlagsHolding - rd.ResourceFlags -= rd.ResourceFlags & resourceFlagsOwnership - rd.ResourceFlags &= ^resourceFlagsEmptyApp - if rd.IsEmptyAppFields() && hadHolding { - rd.ResourceFlags |= resourceFlagsEmptyApp - } -} - -func (rd *resourcesData) SetAppParams(ap basics.AppParams, haveHoldings bool) { - rd.ApprovalProgram = ap.ApprovalProgram - rd.ClearStateProgram = ap.ClearStateProgram - rd.GlobalState = ap.GlobalState - rd.LocalStateSchemaNumUint = ap.LocalStateSchema.NumUint - rd.LocalStateSchemaNumByteSlice = ap.LocalStateSchema.NumByteSlice - rd.GlobalStateSchemaNumUint = ap.GlobalStateSchema.NumUint - rd.GlobalStateSchemaNumByteSlice = ap.GlobalStateSchema.NumByteSlice - rd.ExtraProgramPages = ap.ExtraProgramPages - rd.ResourceFlags |= resourceFlagsOwnership - if !haveHoldings { - rd.ResourceFlags |= resourceFlagsNotHolding - } - rd.ResourceFlags &= ^resourceFlagsEmptyApp - if rd.IsEmptyAppFields() { - rd.ResourceFlags |= resourceFlagsEmptyApp - } -} - -func (rd *resourcesData) GetAppParams() basics.AppParams { - return basics.AppParams{ - ApprovalProgram: rd.ApprovalProgram, - ClearStateProgram: rd.ClearStateProgram, - GlobalState: rd.GlobalState, - StateSchemas: basics.StateSchemas{ - LocalStateSchema: basics.StateSchema{ - NumUint: rd.LocalStateSchemaNumUint, - NumByteSlice: rd.LocalStateSchemaNumByteSlice, - }, - GlobalStateSchema: basics.StateSchema{ - NumUint: rd.GlobalStateSchemaNumUint, - NumByteSlice: rd.GlobalStateSchemaNumByteSlice, - }, - }, - ExtraProgramPages: rd.ExtraProgramPages, - } -} - -func (rd *resourcesData) SetAssetData(ap ledgercore.AssetParamsDelta, ah ledgercore.AssetHoldingDelta) { - if ah.Holding != nil { - rd.SetAssetHolding(*ah.Holding) - } else if ah.Deleted { - rd.ClearAssetHolding() - } - if ap.Params != nil { - rd.SetAssetParams(*ap.Params, rd.IsHolding()) - } else if ap.Deleted { - rd.ClearAssetParams() - } -} - -func (rd *resourcesData) SetAppData(ap ledgercore.AppParamsDelta, al ledgercore.AppLocalStateDelta) { - if al.LocalState != nil { - rd.SetAppLocalState(*al.LocalState) - } else if al.Deleted { - rd.ClearAppLocalState() - } - if ap.Params != nil { - rd.SetAppParams(*ap.Params, rd.IsHolding()) - } else if ap.Deleted { - rd.ClearAppParams() - } -} - func accountDataResources( ctx context.Context, accountData *basics.AccountData, rowid int64, - outputResourceCb func(ctx context.Context, rowid int64, cidx basics.CreatableIndex, rd *resourcesData) error, + outputResourceCb func(ctx context.Context, rowid int64, cidx basics.CreatableIndex, rd *store.ResourcesData) error, ) error { // handle all the assets we can find: for aidx, holding := range accountData.Assets { - var rd resourcesData + var rd store.ResourcesData rd.SetAssetHolding(holding) if ap, has := accountData.AssetParams[aidx]; has { rd.SetAssetParams(ap, true) @@ -2062,7 +1376,7 @@ func accountDataResources( } } for aidx, aparams := range accountData.AssetParams { - var rd resourcesData + var rd store.ResourcesData rd.SetAssetParams(aparams, false) err := outputResourceCb(ctx, rowid, basics.CreatableIndex(aidx), &rd) if err != nil { @@ -2072,7 +1386,7 @@ func accountDataResources( // handle all the applications we can find: for aidx, localState := range accountData.AppLocalStates { - var rd resourcesData + var rd store.ResourcesData rd.SetAppLocalState(localState) if ap, has := accountData.AppParams[aidx]; has { rd.SetAppParams(ap, true) @@ -2084,7 +1398,7 @@ func accountDataResources( } } for aidx, aparams := range accountData.AppParams { - var rd resourcesData + var rd store.ResourcesData rd.SetAppParams(aparams, false) err := outputResourceCb(ctx, rowid, basics.CreatableIndex(aidx), &rd) if err != nil { @@ -2175,7 +1489,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro if err != nil { return err } - var newAccountData baseAccountData + var newAccountData store.BaseAccountData newAccountData.SetAccountData(&accountData) encodedAcctData = protocol.Encode(&newAccountData) @@ -2199,7 +1513,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro if err != nil { return err } - insertResourceCallback := func(ctx context.Context, rowID int64, cidx basics.CreatableIndex, rd *resourcesData) error { + insertResourceCallback := func(ctx context.Context, rowID int64, cidx basics.CreatableIndex, rd *store.ResourcesData) error { var err error if rd != nil { encodedData := protocol.Encode(rd) @@ -2385,9 +1699,9 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre } type acctState struct { - old baseAccountData + old store.BaseAccountData oldEnc []byte - new baseAccountData + new store.BaseAccountData newEnc []byte } acctRehash := make(map[basics.Address]acctState) @@ -2406,7 +1720,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre err = fmt.Errorf("account DB address length mismatch: %d != %d", len(addrbuf), len(addr)) return err } - var ba baseAccountData + var ba store.BaseAccountData err = protocol.Decode(encodedAcctData, &ba) if err != nil { return err @@ -2418,8 +1732,8 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre copy(addr[:], addrbuf) return fmt.Errorf("non valid norm balance for online account %s", addr.String()) } - var baseOnlineAD baseOnlineAccountData - baseOnlineAD.baseVotingData = ba.baseVotingData + var baseOnlineAD store.BaseOnlineAccountData + baseOnlineAD.BaseVotingData = ba.BaseVotingData baseOnlineAD.MicroAlgos = ba.MicroAlgos baseOnlineAD.RewardsBase = ba.RewardsBase encodedOnlineAcctData := protocol.Encode(&baseOnlineAD) @@ -2602,413 +1916,6 @@ func accountsHashRound(ctx context.Context, tx *sql.Tx) (hashrnd basics.Round, e return } -func accountsInitDbQueries(q db.Queryable) (*accountsDbQueries, error) { - var err error - qs := &accountsDbQueries{} - - qs.listCreatablesStmt, err = q.Prepare("SELECT acctrounds.rnd, assetcreators.asset, assetcreators.creator FROM acctrounds LEFT JOIN assetcreators ON assetcreators.asset <= ? AND assetcreators.ctype = ? WHERE acctrounds.id='acctbase' ORDER BY assetcreators.asset desc LIMIT ?") - if err != nil { - return nil, err - } - - qs.lookupStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, accountbase.data FROM acctrounds LEFT JOIN accountbase ON address=? WHERE id='acctbase'") - if err != nil { - return nil, err - } - - qs.lookupResourcesStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, resources.data FROM acctrounds LEFT JOIN accountbase ON accountbase.address = ? LEFT JOIN resources ON accountbase.rowid = resources.addrid AND resources.aidx = ? WHERE id='acctbase'") - if err != nil { - return nil, err - } - - qs.lookupAllResourcesStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, resources.aidx, resources.data FROM acctrounds LEFT JOIN accountbase ON accountbase.address = ? LEFT JOIN resources ON accountbase.rowid = resources.addrid WHERE id='acctbase'") - if err != nil { - return nil, err - } - - qs.lookupKvPairStmt, err = q.Prepare("SELECT acctrounds.rnd, kvstore.value FROM acctrounds LEFT JOIN kvstore ON key = ? WHERE id='acctbase';") - if err != nil { - return nil, err - } - - qs.lookupKeysByRangeStmt, err = q.Prepare("SELECT acctrounds.rnd, kvstore.key FROM acctrounds LEFT JOIN kvstore ON kvstore.key >= ? AND kvstore.key < ? WHERE id='acctbase'") - if err != nil { - return nil, err - } - - qs.lookupCreatorStmt, err = q.Prepare("SELECT acctrounds.rnd, assetcreators.creator FROM acctrounds LEFT JOIN assetcreators ON asset = ? AND ctype = ? WHERE id='acctbase'") - if err != nil { - return nil, err - } - - return qs, nil -} - -func onlineAccountsInitDbQueries(r db.Queryable) (*onlineAccountsDbQueries, error) { - var err error - qs := &onlineAccountsDbQueries{} - - qs.lookupOnlineStmt, err = r.Prepare("SELECT onlineaccounts.rowid, onlineaccounts.updround, acctrounds.rnd, onlineaccounts.data FROM acctrounds LEFT JOIN onlineaccounts ON address=? AND updround <= ? WHERE id='acctbase' ORDER BY updround DESC LIMIT 1") - if err != nil { - return nil, err - } - - qs.lookupOnlineHistoryStmt, err = r.Prepare("SELECT onlineaccounts.rowid, onlineaccounts.updround, acctrounds.rnd, onlineaccounts.data FROM acctrounds LEFT JOIN onlineaccounts ON address=? WHERE id='acctbase' ORDER BY updround ASC") - if err != nil { - return nil, err - } - - qs.lookupOnlineTotalsStmt, err = r.Prepare("SELECT data FROM onlineroundparamstail WHERE rnd=?") - if err != nil { - return nil, err - } - return qs, nil -} - -// listCreatables returns an array of CreatableLocator which have CreatableIndex smaller or equal to maxIdx and are of the provided CreatableType. -func (qs *accountsDbQueries) listCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) { - err = db.Retry(func() error { - // Query for assets in range - rows, err := qs.listCreatablesStmt.Query(maxIdx, ctype, maxResults) - if err != nil { - return err - } - defer rows.Close() - - // For each row, copy into a new CreatableLocator and append to results - var buf []byte - var cl basics.CreatableLocator - var creatableIndex sql.NullInt64 - for rows.Next() { - err = rows.Scan(&dbRound, &creatableIndex, &buf) - if err != nil { - return err - } - if !creatableIndex.Valid { - // we received an entry without any index. This would happen only on the first entry when there are no creatables of the requested type. - break - } - cl.Index = basics.CreatableIndex(creatableIndex.Int64) - copy(cl.Creator[:], buf) - cl.Type = ctype - results = append(results, cl) - } - return nil - }) - return -} - -// sql.go has the following contradictory comments: - -// Reference types such as []byte are only valid until the next call to Scan -// and should not be retained. Their underlying memory is owned by the driver. -// If retention is necessary, copy their values before the next call to Scan. - -// If a dest argument has type *[]byte, Scan saves in that argument a -// copy of the corresponding data. The copy is owned by the caller and -// can be modified and held indefinitely. The copy can be avoided by -// using an argument of type *RawBytes instead; see the documentation -// for RawBytes for restrictions on its use. - -// After check source code, a []byte slice destination is definitely cloned. - -func (qs *accountsDbQueries) lookupKeyValue(key string) (pv persistedKVData, err error) { - err = db.Retry(func() error { - var val []byte - // Cast to []byte to avoid interpretation as character string, see note in upsertKvPair - err := qs.lookupKvPairStmt.QueryRow([]byte(key)).Scan(&pv.round, &val) - if err != nil { - // this should never happen; it indicates that we don't have a current round in the acctrounds table. - if err == sql.ErrNoRows { - // Return the zero value of data - err = fmt.Errorf("unable to query value for key %v : %w", key, err) - } - return err - } - if val != nil { // We got a non-null value, so it exists - pv.value = val - return nil - } - // we don't have that key, just return pv with the database round (pv.value==nil) - return nil - }) - return -} - -// keyPrefixIntervalPreprocessing is implemented to generate an interval for DB queries that look up keys by prefix. -// Such DB query was designed this way, to trigger the binary search optimization in SQLITE3. -// The DB comparison for blob typed primary key is lexicographic, i.e., byte by byte. -// In this way, we can introduce an interval that a primary key should be >= some prefix, < some prefix increment. -// A corner case to consider is that, the prefix has last byte 0xFF, or the prefix is full of 0xFF. -// - The first case can be solved by carrying, e.g., prefix = 0x1EFF -> interval being >= 0x1EFF and < 0x1F -// - The second case can be solved by disregarding the upper limit, i.e., prefix = 0xFFFF -> interval being >= 0xFFFF -// Another corner case to consider is empty byte, []byte{} or nil. -// - In both cases, the results are interval >= "", i.e., returns []byte{} for prefix, and nil for prefixIncr. -func keyPrefixIntervalPreprocessing(prefix []byte) ([]byte, []byte) { - if prefix == nil { - prefix = []byte{} - } - prefixIncr := make([]byte, len(prefix)) - copy(prefixIncr, prefix) - for i := len(prefix) - 1; i >= 0; i-- { - currentByteIncr := int(prefix[i]) + 1 - if currentByteIncr > 0xFF { - prefixIncr = prefixIncr[:len(prefixIncr)-1] - continue - } - prefixIncr[i] = byte(currentByteIncr) - return prefix, prefixIncr - } - return prefix, nil -} - -func (qs *accountsDbQueries) lookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) { - start, end := keyPrefixIntervalPreprocessing([]byte(prefix)) - if end == nil { - // Not an expected use case, it's asking for all keys, or all keys - // prefixed by some number of 0xFF bytes. - return 0, fmt.Errorf("Lookup by strange prefix %#v", prefix) - } - err = db.Retry(func() error { - var rows *sql.Rows - rows, err = qs.lookupKeysByRangeStmt.Query(start, end) - if err != nil { - return err - } - defer rows.Close() - - var v sql.NullString - - for rows.Next() { - if resultCount == maxKeyNum { - return nil - } - err = rows.Scan(&round, &v) - if err != nil { - return err - } - if v.Valid { - if _, ok := results[v.String]; ok { - continue - } - results[v.String] = true - resultCount++ - } - } - return nil - }) - return -} - -func (qs *accountsDbQueries) lookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) { - err = db.Retry(func() error { - var buf []byte - err := qs.lookupCreatorStmt.QueryRow(cidx, ctype).Scan(&dbRound, &buf) - - // this shouldn't happen unless we can't figure the round number. - if err == sql.ErrNoRows { - return fmt.Errorf("lookupCreator was unable to retrieve round number") - } - - // Some other database error - if err != nil { - return err - } - - if len(buf) > 0 { - ok = true - copy(addr[:], buf) - } - return nil - }) - return -} - -func (qs *accountsDbQueries) lookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data persistedResourcesData, err error) { - err = db.Retry(func() error { - var buf []byte - var rowid sql.NullInt64 - err := qs.lookupResourcesStmt.QueryRow(addr[:], aidx).Scan(&rowid, &data.round, &buf) - if err == nil { - data.aidx = aidx - if len(buf) > 0 && rowid.Valid { - data.addrid = rowid.Int64 - err = protocol.Decode(buf, &data.data) - if err != nil { - return err - } - if ctype == basics.AssetCreatable && !data.data.IsAsset() { - return fmt.Errorf("lookupResources asked for an asset but got %v", data.data) - } - if ctype == basics.AppCreatable && !data.data.IsApp() { - return fmt.Errorf("lookupResources asked for an app but got %v", data.data) - } - return nil - } - data.data = makeResourcesData(0) - // we don't have that account, just return the database round. - return nil - } - - // this should never happen; it indicates that we don't have a current round in the acctrounds table. - if err == sql.ErrNoRows { - // Return the zero value of data - return fmt.Errorf("unable to query resource data for address %v aidx %v ctype %v : %w", addr, aidx, ctype, err) - } - return err - }) - return -} - -func (qs *accountsDbQueries) lookupAllResources(addr basics.Address) (data []persistedResourcesData, rnd basics.Round, err error) { - err = db.Retry(func() error { - // Query for all resources - rows, err := qs.lookupAllResourcesStmt.Query(addr[:]) - if err != nil { - return err - } - defer rows.Close() - - var addrid, aidx sql.NullInt64 - var dbRound basics.Round - data = nil - var buf []byte - for rows.Next() { - err := rows.Scan(&addrid, &dbRound, &aidx, &buf) - if err != nil { - return err - } - if !addrid.Valid || !aidx.Valid { - // we received an entry without any index. This would happen only on the first entry when there are no resources for this address. - // ensure this is the first entry, set the round and return - if len(data) != 0 { - return fmt.Errorf("lookupAllResources: unexpected invalid result on non-first resource record: (%v, %v)", addrid.Valid, aidx.Valid) - } - rnd = dbRound - break - } - var resData resourcesData - err = protocol.Decode(buf, &resData) - if err != nil { - return err - } - data = append(data, persistedResourcesData{ - addrid: addrid.Int64, - aidx: basics.CreatableIndex(aidx.Int64), - data: resData, - round: dbRound, - }) - rnd = dbRound - } - return nil - }) - return -} - -// lookup looks up for a the account data given it's address. It returns the persistedAccountData, which includes the current database round and the matching -// account data, if such was found. If no matching account data could be found for the given address, an empty account data would -// be retrieved. -func (qs *accountsDbQueries) lookup(addr basics.Address) (data persistedAccountData, err error) { - err = db.Retry(func() error { - var buf []byte - var rowid sql.NullInt64 - err := qs.lookupStmt.QueryRow(addr[:]).Scan(&rowid, &data.round, &buf) - if err == nil { - data.addr = addr - if len(buf) > 0 && rowid.Valid { - data.rowid = rowid.Int64 - err = protocol.Decode(buf, &data.accountData) - return err - } - // we don't have that account, just return the database round. - return nil - } - - // this should never happen; it indicates that we don't have a current round in the acctrounds table. - if err == sql.ErrNoRows { - // Return the zero value of data - return fmt.Errorf("unable to query account data for address %v : %w", addr, err) - } - - return err - }) - return -} - -func (qs *onlineAccountsDbQueries) lookupOnline(addr basics.Address, rnd basics.Round) (data persistedOnlineAccountData, err error) { - err = db.Retry(func() error { - var buf []byte - var rowid sql.NullInt64 - var updround sql.NullInt64 - err := qs.lookupOnlineStmt.QueryRow(addr[:], rnd).Scan(&rowid, &updround, &data.round, &buf) - if err == nil { - data.addr = addr - if len(buf) > 0 && rowid.Valid && updround.Valid { - data.rowid = rowid.Int64 - data.updRound = basics.Round(updround.Int64) - err = protocol.Decode(buf, &data.accountData) - return err - } - // we don't have that account, just return the database round. - return nil - } - - // this should never happen; it indicates that we don't have a current round in the acctrounds table. - if err == sql.ErrNoRows { - // Return the zero value of data - return fmt.Errorf("unable to query online account data for address %v : %w", addr, err) - } - - return err - }) - return -} - -func (qs *onlineAccountsDbQueries) lookupOnlineTotalsHistory(round basics.Round) (basics.MicroAlgos, error) { - data := ledgercore.OnlineRoundParamsData{} - err := db.Retry(func() error { - row := qs.lookupOnlineTotalsStmt.QueryRow(round) - var buf []byte - err := row.Scan(&buf) - if err != nil { - return err - } - err = protocol.Decode(buf, &data) - if err != nil { - return err - } - return nil - }) - return basics.MicroAlgos{Raw: data.OnlineSupply}, err -} - -func (qs *onlineAccountsDbQueries) lookupOnlineHistory(addr basics.Address) (result []persistedOnlineAccountData, rnd basics.Round, err error) { - err = db.Retry(func() error { - rows, err := qs.lookupOnlineHistoryStmt.Query(addr[:]) - if err != nil { - return err - } - defer rows.Close() - - for rows.Next() { - var buf []byte - data := persistedOnlineAccountData{} - err := rows.Scan(&data.rowid, &data.updRound, &rnd, &buf) - if err != nil { - return err - } - err = protocol.Decode(buf, &data.accountData) - if err != nil { - return err - } - data.addr = addr - result = append(result, data) - } - return err - }) - return -} - func storeCatchpoint(ctx context.Context, e db.Executable, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) { err = db.Retry(func() (err error) { query := "DELETE FROM storedcatchpoints WHERE round=?" @@ -3125,37 +2032,6 @@ func deleteCatchpointStateImpl(ctx context.Context, e db.Executable, stateName c return err } -func (qs *accountsDbQueries) close() { - preparedQueries := []**sql.Stmt{ - &qs.listCreatablesStmt, - &qs.lookupStmt, - &qs.lookupResourcesStmt, - &qs.lookupAllResourcesStmt, - &qs.lookupKvPairStmt, - &qs.lookupKeysByRangeStmt, - &qs.lookupCreatorStmt, - } - for _, preparedQuery := range preparedQueries { - if (*preparedQuery) != nil { - (*preparedQuery).Close() - *preparedQuery = nil - } - } -} - -func (qs *onlineAccountsDbQueries) close() { - preparedQueries := []**sql.Stmt{ - &qs.lookupOnlineStmt, - &qs.lookupOnlineHistoryStmt, - } - for _, preparedQuery := range preparedQueries { - if (*preparedQuery) != nil { - (*preparedQuery).Close() - *preparedQuery = nil - } - } -} - // accountsOnlineTop returns the top n online accounts starting at position offset // (that is, the top offset'th account through the top offset+n-1'th account). // @@ -3191,7 +2067,7 @@ ORDER BY normalizedonlinebalance DESC, address DESC LIMIT ? OFFSET ?`, rnd, n, o return nil, err } - var data baseOnlineAccountData + var data store.BaseOnlineAccountData err = protocol.Decode(buf, &data) if err != nil { return nil, err @@ -3220,26 +2096,26 @@ ORDER BY normalizedonlinebalance DESC, address DESC LIMIT ? OFFSET ?`, rnd, n, o return res, rows.Err() } -func onlineAccountsAll(tx *sql.Tx, maxAccounts uint64) ([]persistedOnlineAccountData, error) { +func onlineAccountsAll(tx *sql.Tx, maxAccounts uint64) ([]store.PersistedOnlineAccountData, error) { rows, err := tx.Query("SELECT rowid, address, updround, data FROM onlineaccounts ORDER BY address, updround ASC") if err != nil { return nil, err } defer rows.Close() - result := make([]persistedOnlineAccountData, 0, maxAccounts) + result := make([]store.PersistedOnlineAccountData, 0, maxAccounts) var numAccounts uint64 seenAddr := make([]byte, len(basics.Address{})) for rows.Next() { var addrbuf []byte var buf []byte - data := persistedOnlineAccountData{} - err := rows.Scan(&data.rowid, &addrbuf, &data.updRound, &buf) + data := store.PersistedOnlineAccountData{} + err := rows.Scan(&data.Rowid, &addrbuf, &data.UpdRound, &buf) if err != nil { return nil, err } - if len(addrbuf) != len(data.addr) { - err = fmt.Errorf("account DB address length mismatch: %d != %d", len(addrbuf), len(data.addr)) + if len(addrbuf) != len(data.Addr) { + err = fmt.Errorf("account DB address length mismatch: %d != %d", len(addrbuf), len(data.Addr)) return nil, err } if maxAccounts > 0 { @@ -3251,8 +2127,8 @@ func onlineAccountsAll(tx *sql.Tx, maxAccounts uint64) ([]persistedOnlineAccount copy(seenAddr, addrbuf) } } - copy(data.addr[:], addrbuf) - err = protocol.Decode(buf, &data.accountData) + copy(data.Addr[:], addrbuf) + err = protocol.Decode(buf, &data.AccountData) if err != nil { return nil, err } @@ -3336,269 +2212,22 @@ func accountsPruneOnlineRoundParams(tx *sql.Tx, deleteBeforeRound basics.Round) return err } -type accountsWriter interface { - insertAccount(addr basics.Address, normBalance uint64, data baseAccountData) (rowid int64, err error) - deleteAccount(rowid int64) (rowsAffected int64, err error) - updateAccount(rowid int64, normBalance uint64, data baseAccountData) (rowsAffected int64, err error) - - insertResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowid int64, err error) - deleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) - updateResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowsAffected int64, err error) - - upsertKvPair(key string, value []byte) error - deleteKvPair(key string) error - - insertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) - deleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) - - close() -} - -type onlineAccountsWriter interface { - insertOnlineAccount(addr basics.Address, normBalance uint64, data baseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) - - close() -} - -type accountsSQLWriter struct { - insertCreatableIdxStmt, deleteCreatableIdxStmt *sql.Stmt - deleteByRowIDStmt, insertStmt, updateStmt *sql.Stmt - deleteResourceStmt, insertResourceStmt, updateResourceStmt *sql.Stmt - deleteKvPairStmt, upsertKvPairStmt *sql.Stmt -} - -type onlineAccountsSQLWriter struct { - insertStmt, updateStmt *sql.Stmt -} - -func (w *accountsSQLWriter) close() { - // Formatted to match the type definition above - preparedStmts := []**sql.Stmt{ - &w.insertCreatableIdxStmt, &w.deleteCreatableIdxStmt, - &w.deleteByRowIDStmt, &w.insertStmt, &w.updateStmt, - &w.deleteResourceStmt, &w.insertResourceStmt, &w.updateResourceStmt, - &w.deleteKvPairStmt, &w.upsertKvPairStmt, - } - - for _, stmt := range preparedStmts { - if (*stmt) != nil { - (*stmt).Close() - *stmt = nil - } - } - -} - -func (w *onlineAccountsSQLWriter) close() { - if w.insertStmt != nil { - w.insertStmt.Close() - w.insertStmt = nil - } -} - -func makeAccountsSQLWriter(tx *sql.Tx, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) { - w = new(accountsSQLWriter) - - if hasAccounts { - w.deleteByRowIDStmt, err = tx.Prepare("DELETE FROM accountbase WHERE rowid=?") - if err != nil { - return - } - - w.insertStmt, err = tx.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)") - if err != nil { - return - } - - w.updateStmt, err = tx.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?") - if err != nil { - return - } - } - - if hasResources { - w.deleteResourceStmt, err = tx.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?") - if err != nil { - return - } - - w.insertResourceStmt, err = tx.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") - if err != nil { - return - } - - w.updateResourceStmt, err = tx.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?") - if err != nil { - return - } - } - - if hasKvPairs { - w.upsertKvPairStmt, err = tx.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value") - if err != nil { - return - } - - w.deleteKvPairStmt, err = tx.Prepare("DELETE FROM kvstore WHERE key=?") - if err != nil { - return - } - } - - if hasCreatables { - w.insertCreatableIdxStmt, err = tx.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)") - if err != nil { - return - } - - w.deleteCreatableIdxStmt, err = tx.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?") - if err != nil { - return - } - } - return -} - -func (w accountsSQLWriter) insertAccount(addr basics.Address, normBalance uint64, data baseAccountData) (rowid int64, err error) { - result, err := w.insertStmt.Exec(addr[:], normBalance, protocol.Encode(&data)) - if err != nil { - return - } - rowid, err = result.LastInsertId() - return -} - -func (w accountsSQLWriter) deleteAccount(rowid int64) (rowsAffected int64, err error) { - result, err := w.deleteByRowIDStmt.Exec(rowid) - if err != nil { - return - } - rowsAffected, err = result.RowsAffected() - return -} - -func (w accountsSQLWriter) updateAccount(rowid int64, normBalance uint64, data baseAccountData) (rowsAffected int64, err error) { - result, err := w.updateStmt.Exec(normBalance, protocol.Encode(&data), rowid) - if err != nil { - return - } - rowsAffected, err = result.RowsAffected() - return -} - -func (w accountsSQLWriter) insertResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowid int64, err error) { - result, err := w.insertResourceStmt.Exec(addrid, aidx, protocol.Encode(&data)) - if err != nil { - return - } - rowid, err = result.LastInsertId() - return -} - -func (w accountsSQLWriter) deleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) { - result, err := w.deleteResourceStmt.Exec(addrid, aidx) - if err != nil { - return - } - rowsAffected, err = result.RowsAffected() - return -} - -func (w accountsSQLWriter) updateResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowsAffected int64, err error) { - result, err := w.updateResourceStmt.Exec(protocol.Encode(&data), addrid, aidx) - if err != nil { - return - } - rowsAffected, err = result.RowsAffected() - return -} - -func (w accountsSQLWriter) upsertKvPair(key string, value []byte) error { - // NOTE! If we are passing in `string`, then for `BoxKey` case, - // we might contain 0-byte in boxKey, coming from uint64 appID. - // The consequence would be DB key write in be cut off after such 0-byte. - // Casting `string` to `[]byte` avoids such trouble, and test: - // - `TestBoxNamesByAppIDs` in `acctupdates_test` - // relies on such modification. - result, err := w.upsertKvPairStmt.Exec([]byte(key), value) - if err != nil { - return err - } - _, err = result.LastInsertId() - return err -} - -func (w accountsSQLWriter) deleteKvPair(key string) error { - // Cast to []byte to avoid interpretation as character string, see note in upsertKvPair - result, err := w.deleteKvPairStmt.Exec([]byte(key)) - if err != nil { - return err - } - _, err = result.RowsAffected() - return err -} - -func (w accountsSQLWriter) insertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) { - result, err := w.insertCreatableIdxStmt.Exec(cidx, creator, ctype) - if err != nil { - return - } - rowid, err = result.LastInsertId() - return -} - -func (w accountsSQLWriter) deleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { - result, err := w.deleteCreatableIdxStmt.Exec(cidx, ctype) - if err != nil { - return - } - rowsAffected, err = result.RowsAffected() - return -} - -func makeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) { - w = new(onlineAccountsSQLWriter) - - if hasAccounts { - w.insertStmt, err = tx.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return - } - - w.updateStmt, err = tx.Prepare("UPDATE onlineaccounts SET normalizedonlinebalance = ?, data = ?, updround = ?, votelastvalid =? WHERE rowid = ?") - if err != nil { - return - } - } - - return -} - -func (w onlineAccountsSQLWriter) insertOnlineAccount(addr basics.Address, normBalance uint64, data baseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) { - result, err := w.insertStmt.Exec(addr[:], normBalance, protocol.Encode(&data), updRound, voteLastValid) - if err != nil { - return - } - rowid, err = result.LastInsertId() - return -} - // accountsNewRound is a convenience wrapper for accountsNewRoundImpl func accountsNewRound( tx *sql.Tx, updates compactAccountDeltas, resources compactResourcesDeltas, kvPairs map[string]modifiedKvValue, creatables map[basics.CreatableIndex]ledgercore.ModifiedCreatable, proto config.ConsensusParams, lastUpdateRound basics.Round, -) (updatedAccounts []persistedAccountData, updatedResources map[basics.Address][]persistedResourcesData, updatedKVs map[string]persistedKVData, err error) { +) (updatedAccounts []store.PersistedAccountData, updatedResources map[basics.Address][]store.PersistedResourcesData, updatedKVs map[string]store.PersistedKVData, err error) { hasAccounts := updates.len() > 0 hasResources := resources.len() > 0 hasKvPairs := len(kvPairs) > 0 hasCreatables := len(creatables) > 0 - writer, err := makeAccountsSQLWriter(tx, hasAccounts, hasResources, hasKvPairs, hasCreatables) + writer, err := store.MakeAccountsSQLWriter(tx, hasAccounts, hasResources, hasKvPairs, hasCreatables) if err != nil { return } - defer writer.close() + defer writer.Close() return accountsNewRoundImpl(writer, updates, resources, kvPairs, creatables, proto, lastUpdateRound) } @@ -3607,14 +2236,14 @@ func onlineAccountsNewRound( tx *sql.Tx, updates compactOnlineAccountDeltas, proto config.ConsensusParams, lastUpdateRound basics.Round, -) (updatedAccounts []persistedOnlineAccountData, err error) { +) (updatedAccounts []store.PersistedOnlineAccountData, err error) { hasAccounts := updates.len() > 0 - writer, err := makeOnlineAccountsSQLWriter(tx, hasAccounts) + writer, err := store.MakeOnlineAccountsSQLWriter(tx, hasAccounts) if err != nil { return } - defer writer.close() + defer writer.Close() updatedAccounts, err = onlineAccountsNewRoundImpl(writer, updates, proto, lastUpdateRound) return @@ -3623,16 +2252,16 @@ func onlineAccountsNewRound( // accountsNewRoundImpl updates the accountbase and assetcreators tables by applying the provided deltas to the accounts / creatables. // The function returns a persistedAccountData for the modified accounts which can be stored in the base cache. func accountsNewRoundImpl( - writer accountsWriter, + writer store.AccountsWriter, updates compactAccountDeltas, resources compactResourcesDeltas, kvPairs map[string]modifiedKvValue, creatables map[basics.CreatableIndex]ledgercore.ModifiedCreatable, proto config.ConsensusParams, lastUpdateRound basics.Round, -) (updatedAccounts []persistedAccountData, updatedResources map[basics.Address][]persistedResourcesData, updatedKVs map[string]persistedKVData, err error) { - updatedAccounts = make([]persistedAccountData, updates.len()) +) (updatedAccounts []store.PersistedAccountData, updatedResources map[basics.Address][]store.PersistedResourcesData, updatedKVs map[string]store.PersistedKVData, err error) { + updatedAccounts = make([]store.PersistedAccountData, updates.len()) updatedAccountIdx := 0 newAddressesRowIDs := make(map[basics.Address]int64) for i := 0; i < updates.len(); i++ { data := updates.getByIdx(i) - if data.oldAcct.rowid == 0 { + if data.oldAcct.Rowid == 0 { // zero rowid means we don't have a previous value. if data.newAcct.IsEmpty() { // IsEmpty means we don't have a previous value. Note, can't use newAcct.MsgIsZero @@ -3642,10 +2271,10 @@ func accountsNewRoundImpl( // create a new entry. var rowid int64 normBalance := data.newAcct.NormalizedOnlineBalance(proto) - rowid, err = writer.insertAccount(data.address, normBalance, data.newAcct) + rowid, err = writer.InsertAccount(data.address, normBalance, data.newAcct) if err == nil { - updatedAccounts[updatedAccountIdx].rowid = rowid - updatedAccounts[updatedAccountIdx].accountData = data.newAcct + updatedAccounts[updatedAccountIdx].Rowid = rowid + updatedAccounts[updatedAccountIdx].AccountData = data.newAcct newAddressesRowIDs[data.address] = rowid } } @@ -3654,25 +2283,25 @@ func accountsNewRoundImpl( if data.newAcct.IsEmpty() { // new value is zero, which means we need to delete the current value. var rowsAffected int64 - rowsAffected, err = writer.deleteAccount(data.oldAcct.rowid) + rowsAffected, err = writer.DeleteAccount(data.oldAcct.Rowid) if err == nil { // we deleted the entry successfully. - updatedAccounts[updatedAccountIdx].rowid = 0 - updatedAccounts[updatedAccountIdx].accountData = baseAccountData{} + updatedAccounts[updatedAccountIdx].Rowid = 0 + updatedAccounts[updatedAccountIdx].AccountData = store.BaseAccountData{} if rowsAffected != 1 { - err = fmt.Errorf("failed to delete accountbase row for account %v, rowid %d", data.address, data.oldAcct.rowid) + err = fmt.Errorf("failed to delete accountbase row for account %v, rowid %d", data.address, data.oldAcct.Rowid) } } } else { var rowsAffected int64 normBalance := data.newAcct.NormalizedOnlineBalance(proto) - rowsAffected, err = writer.updateAccount(data.oldAcct.rowid, normBalance, data.newAcct) + rowsAffected, err = writer.UpdateAccount(data.oldAcct.Rowid, normBalance, data.newAcct) if err == nil { // rowid doesn't change on update. - updatedAccounts[updatedAccountIdx].rowid = data.oldAcct.rowid - updatedAccounts[updatedAccountIdx].accountData = data.newAcct + updatedAccounts[updatedAccountIdx].Rowid = data.oldAcct.Rowid + updatedAccounts[updatedAccountIdx].AccountData = data.newAcct if rowsAffected != 1 { - err = fmt.Errorf("failed to update accountbase row for account %v, rowid %d", data.address, data.oldAcct.rowid) + err = fmt.Errorf("failed to update accountbase row for account %v, rowid %d", data.address, data.oldAcct.Rowid) } } } @@ -3683,12 +2312,12 @@ func accountsNewRoundImpl( } // set the returned persisted account states so that we could store that as the baseAccounts in commitRound - updatedAccounts[updatedAccountIdx].round = lastUpdateRound - updatedAccounts[updatedAccountIdx].addr = data.address + updatedAccounts[updatedAccountIdx].Round = lastUpdateRound + updatedAccounts[updatedAccountIdx].Addr = data.address updatedAccountIdx++ } - updatedResources = make(map[basics.Address][]persistedResourcesData) + updatedResources = make(map[basics.Address][]store.PersistedResourcesData) // the resources update is going to be made in three parts: // on the first loop, we will find out all the entries that need to be deleted, and parepare a pendingResourcesDeletion map. @@ -3705,15 +2334,15 @@ func accountsNewRoundImpl( var pendingResourcesDeletion map[resourceKey]struct{} // map to indicate which resources need to be deleted for i := 0; i < resources.len(); i++ { data := resources.getByIdx(i) - if data.oldResource.addrid == 0 || data.oldResource.data.IsEmpty() || !data.newResource.IsEmpty() { + if data.oldResource.Addrid == 0 || data.oldResource.Data.IsEmpty() || !data.newResource.IsEmpty() { continue } if pendingResourcesDeletion == nil { pendingResourcesDeletion = make(map[resourceKey]struct{}) } - pendingResourcesDeletion[resourceKey{addrid: data.oldResource.addrid, aidx: data.oldResource.aidx}] = struct{}{} + pendingResourcesDeletion[resourceKey{addrid: data.oldResource.Addrid, aidx: data.oldResource.Aidx}] = struct{}{} - entry := persistedResourcesData{addrid: 0, aidx: data.oldResource.aidx, data: makeResourcesData(0), round: lastUpdateRound} + entry := store.PersistedResourcesData{Addrid: 0, Aidx: data.oldResource.Aidx, Data: store.MakeResourcesData(0), Round: lastUpdateRound} deltas := updatedResources[data.address] deltas = append(deltas, entry) updatedResources[data.address] = deltas @@ -3722,21 +2351,21 @@ func accountsNewRoundImpl( for i := 0; i < resources.len(); i++ { data := resources.getByIdx(i) addr := data.address - aidx := data.oldResource.aidx - addrid := data.oldResource.addrid + aidx := data.oldResource.Aidx + addrid := data.oldResource.Addrid if addrid == 0 { // new entry, data.oldResource does not have addrid // check if this delta is part of in-memory only account // that is created, funded, transferred, and closed within a commit range - inMemEntry := data.oldResource.data.IsEmpty() && data.newResource.IsEmpty() + inMemEntry := data.oldResource.Data.IsEmpty() && data.newResource.IsEmpty() addrid = newAddressesRowIDs[addr] if addrid == 0 && !inMemEntry { err = fmt.Errorf("cannot resolve address %s (%d), aidx %d, data %v", addr.String(), addrid, aidx, data.newResource) return } } - var entry persistedResourcesData - if data.oldResource.data.IsEmpty() { + var entry store.PersistedResourcesData + if data.oldResource.Data.IsEmpty() { // IsEmpty means we don't have a previous value. Note, can't use oldResource.data.MsgIsZero // because of possibility of empty asset holdings or app local state after opting in, // as well as non-zero UpdateRound field in a new delta @@ -3744,7 +2373,7 @@ func accountsNewRoundImpl( // if we didn't had it before, and we don't have anything now, just skip it. // set zero addrid to mark this entry invalid for subsequent addr to addrid resolution // because the base account might gone. - entry = persistedResourcesData{addrid: 0, aidx: aidx, data: makeResourcesData(0), round: lastUpdateRound} + entry = store.PersistedResourcesData{Addrid: 0, Aidx: aidx, Data: store.MakeResourcesData(0), Round: lastUpdateRound} } else { // create a new entry. if !data.newResource.IsApp() && !data.newResource.IsAsset() { @@ -3758,19 +2387,19 @@ func accountsNewRoundImpl( // update the database entry instead of deleting + inserting. delete(pendingResourcesDeletion, resourceKey{addrid: addrid, aidx: aidx}) var rowsAffected int64 - rowsAffected, err = writer.updateResource(addrid, aidx, data.newResource) + rowsAffected, err = writer.UpdateResource(addrid, aidx, data.newResource) if err == nil { // rowid doesn't change on update. - entry = persistedResourcesData{addrid: addrid, aidx: aidx, data: data.newResource, round: lastUpdateRound} + entry = store.PersistedResourcesData{Addrid: addrid, Aidx: aidx, Data: data.newResource, Round: lastUpdateRound} if rowsAffected != 1 { err = fmt.Errorf("failed to update resources row for addr %s (%d), aidx %d", addr, addrid, aidx) } } } else { - _, err = writer.insertResource(addrid, aidx, data.newResource) + _, err = writer.InsertResource(addrid, aidx, data.newResource) if err == nil { // set the returned persisted account states so that we could store that as the baseResources in commitRound - entry = persistedResourcesData{addrid: addrid, aidx: aidx, data: data.newResource, round: lastUpdateRound} + entry = store.PersistedResourcesData{Addrid: addrid, Aidx: aidx, Data: data.newResource, Round: lastUpdateRound} } } } @@ -3786,10 +2415,10 @@ func accountsNewRoundImpl( return } var rowsAffected int64 - rowsAffected, err = writer.updateResource(addrid, aidx, data.newResource) + rowsAffected, err = writer.UpdateResource(addrid, aidx, data.newResource) if err == nil { // rowid doesn't change on update. - entry = persistedResourcesData{addrid: addrid, aidx: aidx, data: data.newResource, round: lastUpdateRound} + entry = store.PersistedResourcesData{Addrid: addrid, Aidx: aidx, Data: data.newResource, Round: lastUpdateRound} if rowsAffected != 1 { err = fmt.Errorf("failed to update resources row for addr %s (%d), aidx %d", addr, addrid, aidx) } @@ -3810,7 +2439,7 @@ func accountsNewRoundImpl( for delRes := range pendingResourcesDeletion { // new value is zero, which means we need to delete the current value. var rowsAffected int64 - rowsAffected, err = writer.deleteResource(delRes.addrid, delRes.aidx) + rowsAffected, err = writer.DeleteResource(delRes.addrid, delRes.aidx) if err == nil { // we deleted the entry successfully. // set zero addrid to mark this entry invalid for subsequent addr to addrid resolution @@ -3824,21 +2453,21 @@ func accountsNewRoundImpl( } } - updatedKVs = make(map[string]persistedKVData, len(kvPairs)) + updatedKVs = make(map[string]store.PersistedKVData, len(kvPairs)) for key, mv := range kvPairs { if mv.data != nil { // reminder: check oldData for nil here, b/c bytes.Equal conflates nil and "". if mv.oldData != nil && bytes.Equal(mv.oldData, mv.data) { continue // changed back within the delta span } - err = writer.upsertKvPair(key, mv.data) - updatedKVs[key] = persistedKVData{value: mv.data, round: lastUpdateRound} + err = writer.UpsertKvPair(key, mv.data) + updatedKVs[key] = store.PersistedKVData{Value: mv.data, Round: lastUpdateRound} } else { if mv.oldData == nil { // Came and went within the delta span continue } - err = writer.deleteKvPair(key) - updatedKVs[key] = persistedKVData{value: nil, round: lastUpdateRound} + err = writer.DeleteKvPair(key) + updatedKVs[key] = store.PersistedKVData{Value: nil, Round: lastUpdateRound} } if err != nil { return @@ -3847,9 +2476,9 @@ func accountsNewRoundImpl( for cidx, cdelta := range creatables { if cdelta.Created { - _, err = writer.insertCreatable(cidx, cdelta.Ctype, cdelta.Creator[:]) + _, err = writer.InsertCreatable(cidx, cdelta.Ctype, cdelta.Creator[:]) } else { - _, err = writer.deleteCreatable(cidx, cdelta.Ctype) + _, err = writer.DeleteCreatable(cidx, cdelta.Ctype) } if err != nil { return @@ -3860,9 +2489,9 @@ func accountsNewRoundImpl( } func onlineAccountsNewRoundImpl( - writer onlineAccountsWriter, updates compactOnlineAccountDeltas, + writer store.OnlineAccountsWriter, updates compactOnlineAccountDeltas, proto config.ConsensusParams, lastUpdateRound basics.Round, -) (updatedAccounts []persistedOnlineAccountData, err error) { +) (updatedAccounts []store.PersistedOnlineAccountData, err error) { for i := 0; i < updates.len(); i++ { data := updates.getByIdx(i) @@ -3871,7 +2500,7 @@ func onlineAccountsNewRoundImpl( newAcct := data.newAcct[j] updRound := data.updRound[j] newStatus := data.newStatus[j] - if prevAcct.rowid == 0 { + if prevAcct.Rowid == 0 { // zero rowid means we don't have a previous value. if newAcct.IsEmpty() { // IsEmpty means we don't have a previous value. @@ -3884,14 +2513,14 @@ func onlineAccountsNewRoundImpl( // create a new entry. var rowid int64 normBalance := newAcct.NormalizedOnlineBalance(proto) - rowid, err = writer.insertOnlineAccount(data.address, normBalance, newAcct, updRound, uint64(newAcct.VoteLastValid)) + rowid, err = writer.InsertOnlineAccount(data.address, normBalance, newAcct, updRound, uint64(newAcct.VoteLastValid)) if err == nil { - updated := persistedOnlineAccountData{ - addr: data.address, - accountData: newAcct, - round: lastUpdateRound, - rowid: rowid, - updRound: basics.Round(updRound), + updated := store.PersistedOnlineAccountData{ + Addr: data.address, + AccountData: newAcct, + Round: lastUpdateRound, + Rowid: rowid, + UpdRound: basics.Round(updRound), } updatedAccounts = append(updatedAccounts, updated) prevAcct = updated @@ -3909,14 +2538,14 @@ func onlineAccountsNewRoundImpl( err = fmt.Errorf("empty voting data but online account %s: %v", data.address.String(), newAcct) } else { var rowid int64 - rowid, err = writer.insertOnlineAccount(data.address, 0, baseOnlineAccountData{}, updRound, 0) + rowid, err = writer.InsertOnlineAccount(data.address, 0, store.BaseOnlineAccountData{}, updRound, 0) if err == nil { - updated := persistedOnlineAccountData{ - addr: data.address, - accountData: baseOnlineAccountData{}, - round: lastUpdateRound, - rowid: rowid, - updRound: basics.Round(updRound), + updated := store.PersistedOnlineAccountData{ + Addr: data.address, + AccountData: store.BaseOnlineAccountData{}, + Round: lastUpdateRound, + Rowid: rowid, + UpdRound: basics.Round(updRound), } updatedAccounts = append(updatedAccounts, updated) @@ -3924,17 +2553,17 @@ func onlineAccountsNewRoundImpl( } } } else { - if prevAcct.accountData != newAcct { + if prevAcct.AccountData != newAcct { var rowid int64 normBalance := newAcct.NormalizedOnlineBalance(proto) - rowid, err = writer.insertOnlineAccount(data.address, normBalance, newAcct, updRound, uint64(newAcct.VoteLastValid)) + rowid, err = writer.InsertOnlineAccount(data.address, normBalance, newAcct, updRound, uint64(newAcct.VoteLastValid)) if err == nil { - updated := persistedOnlineAccountData{ - addr: data.address, - accountData: newAcct, - round: lastUpdateRound, - rowid: rowid, - updRound: basics.Round(updRound), + updated := store.PersistedOnlineAccountData{ + Addr: data.address, + AccountData: newAcct, + Round: lastUpdateRound, + Rowid: rowid, + UpdRound: basics.Round(updRound), } updatedAccounts = append(updatedAccounts, updated) @@ -4039,7 +2668,7 @@ func onlineAccountsDelete(tx *sql.Tx, forgetBefore basics.Round) (err error) { // reset the state prevAddr = addrbuf - var oad baseOnlineAccountData + var oad store.BaseOnlineAccountData err = protocol.Decode(buf, &oad) if err != nil { return @@ -4299,9 +2928,9 @@ func (iterator *encodedAccountsBatchIter) Next(ctx context.Context, tx *sql.Tx, // gather up to accountCount encoded accounts. bals = make([]encodedBalanceRecordV6, 0, accountCount) var encodedRecord encodedBalanceRecordV6 - var baseAcct baseAccountData + var baseAcct store.BaseAccountData var numAcct int - baseCb := func(addr basics.Address, rowid int64, accountData *baseAccountData, encodedAccountData []byte) (err error) { + baseCb := func(addr basics.Address, rowid int64, accountData *store.BaseAccountData, encodedAccountData []byte) (err error) { encodedRecord = encodedBalanceRecordV6{Address: addr, AccountData: encodedAccountData} baseAcct = *accountData numAcct++ @@ -4311,7 +2940,7 @@ func (iterator *encodedAccountsBatchIter) Next(ctx context.Context, tx *sql.Tx, var totalResources int // emptyCount := 0 - resCb := func(addr basics.Address, cidx basics.CreatableIndex, resData *resourcesData, encodedResourceData []byte, lastResource bool) error { + resCb := func(addr basics.Address, cidx basics.CreatableIndex, resData *store.ResourcesData, encodedResourceData []byte, lastResource bool) error { emptyBaseAcct := baseAcct.TotalAppParams == 0 && baseAcct.TotalAppLocalStates == 0 && baseAcct.TotalAssetParams == 0 && baseAcct.TotalAssets == 0 if !emptyBaseAcct && resData != nil { @@ -4451,7 +3080,7 @@ func makeOrderedAccountsIter(tx *sql.Tx, accountCount int) *orderedAccountsIter type pendingBaseRow struct { addr basics.Address rowid int64 - accountData *baseAccountData + accountData *store.BaseAccountData encodedAccountData []byte } @@ -4463,8 +3092,8 @@ type pendingResourceRow struct { func processAllResources( resRows *sql.Rows, - addr basics.Address, accountData *baseAccountData, acctRowid int64, pr pendingResourceRow, resourceCount int, - callback func(addr basics.Address, creatableIdx basics.CreatableIndex, resData *resourcesData, encodedResourceData []byte, lastResource bool) error, + addr basics.Address, accountData *store.BaseAccountData, acctRowid int64, pr pendingResourceRow, resourceCount int, + callback func(addr basics.Address, creatableIdx basics.CreatableIndex, resData *store.ResourcesData, encodedResourceData []byte, lastResource bool) error, ) (pendingResourceRow, int, error) { var err error count := 0 @@ -4474,7 +3103,7 @@ func processAllResources( var buf []byte var addrid int64 var aidx basics.CreatableIndex - var resData resourcesData + var resData store.ResourcesData for { if pr.addrid != 0 { // some accounts may not have resources, consider the following case: @@ -4513,7 +3142,7 @@ func processAllResources( return pendingResourceRow{addrid, aidx, buf}, count, err } } - resData = resourcesData{} + resData = store.ResourcesData{} err = protocol.Decode(buf, &resData) if err != nil { return pendingResourceRow{}, count, err @@ -4535,8 +3164,8 @@ func processAllResources( func processAllBaseAccountRecords( baseRows *sql.Rows, resRows *sql.Rows, - baseCb func(addr basics.Address, rowid int64, accountData *baseAccountData, encodedAccountData []byte) error, - resCb func(addr basics.Address, creatableIdx basics.CreatableIndex, resData *resourcesData, encodedResourceData []byte, lastResource bool) error, + baseCb func(addr basics.Address, rowid int64, accountData *store.BaseAccountData, encodedAccountData []byte) error, + resCb func(addr basics.Address, creatableIdx basics.CreatableIndex, resData *store.ResourcesData, encodedResourceData []byte, lastResource bool) error, pendingBase pendingBaseRow, pendingResource pendingResourceRow, accountCount int, resourceCount int, ) (int, pendingBaseRow, pendingResourceRow, error) { var addr basics.Address @@ -4544,7 +3173,7 @@ func processAllBaseAccountRecords( var err error count := 0 - var accountData baseAccountData + var accountData store.BaseAccountData var addrbuf []byte var buf []byte var rowid int64 @@ -4572,7 +3201,7 @@ func processAllBaseAccountRecords( copy(addr[:], addrbuf) - accountData = baseAccountData{} + accountData = store.BaseAccountData{} err = protocol.Decode(buf, &accountData) if err != nil { return 0, pendingBaseRow{}, pendingResourceRow{}, err @@ -4615,7 +3244,7 @@ func processAllBaseAccountRecords( } // loadFullAccount converts baseAccountData into basics.AccountData and loads all resources as needed -func loadFullAccount(ctx context.Context, tx *sql.Tx, resourcesTable string, addr basics.Address, addrid int64, data baseAccountData) (ad basics.AccountData, err error) { +func loadFullAccount(ctx context.Context, tx *sql.Tx, resourcesTable string, addr basics.Address, addrid int64, data store.BaseAccountData) (ad basics.AccountData, err error) { ad = data.GetAccountData() hasResources := false @@ -4655,12 +3284,12 @@ func loadFullAccount(ctx context.Context, tx *sql.Tx, resourcesTable string, add if err != nil { return } - var resData resourcesData + var resData store.ResourcesData err = protocol.Decode(buf, &resData) if err != nil { return } - if resData.ResourceFlags == resourceFlagsNotHolding { + if resData.ResourceFlags == store.ResourceFlagsNotHolding { err = fmt.Errorf("addr %s (%d) aidx = %d resourceFlagsNotHolding should not be persisted", addr.String(), addrid, aidx) return } @@ -4729,7 +3358,7 @@ func LoadAllFullAccounts( return } - var data baseAccountData + var data store.BaseAccountData err = protocol.Decode(buf, &data) if err != nil { return @@ -4810,7 +3439,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAd } if iterator.step == oaiStepInsertAccountData { var lastAddrID int64 - baseCb := func(addr basics.Address, rowid int64, accountData *baseAccountData, encodedAccountData []byte) (err error) { + baseCb := func(addr basics.Address, rowid int64, accountData *store.BaseAccountData, encodedAccountData []byte) (err error) { hash := accountHashBuilderV6(addr, accountData, encodedAccountData) _, err = iterator.insertStmt.ExecContext(ctx, rowid, hash) if err != nil { @@ -4820,7 +3449,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []accountAd return nil } - resCb := func(addr basics.Address, cidx basics.CreatableIndex, resData *resourcesData, encodedResourceData []byte, lastResource bool) error { + resCb := func(addr basics.Address, cidx basics.CreatableIndex, resData *store.ResourcesData, encodedResourceData []byte, lastResource bool) error { if resData != nil { hash, err := resourcesHashBuilderV6(resData, addr, cidx, resData.UpdateRound, encodedResourceData) if err != nil { @@ -5020,30 +3649,6 @@ func (iterator *catchpointPendingHashesIterator) Close() { } } -// before compares the round numbers of two persistedAccountData and determines if the current persistedAccountData -// happened before the other. -func (pac *persistedAccountData) before(other *persistedAccountData) bool { - return pac.round < other.round -} - -// before compares the round numbers of two persistedResourcesData and determines if the current persistedResourcesData -// happened before the other. -func (prd *persistedResourcesData) before(other *persistedResourcesData) bool { - return prd.round < other.round -} - -// before compares the round numbers of two persistedKVData and determines if the current persistedKVData -// happened before the other. -func (prd persistedKVData) before(other *persistedKVData) bool { - return prd.round < other.round -} - -// before compares the round numbers of two persistedAccountData and determines if the current persistedAccountData -// happened before the other. -func (pac *persistedOnlineAccountData) before(other *persistedOnlineAccountData) bool { - return pac.updRound < other.updRound -} - // txTailRoundLease is used as part of txTailRound for storing // a single lease. type txTailRoundLease struct { diff --git a/ledger/accountdb_test.go b/ledger/accountdb_test.go index 781c7300c8..5daa191eb2 100644 --- a/ledger/accountdb_test.go +++ b/ledger/accountdb_test.go @@ -21,13 +21,11 @@ import ( "context" "database/sql" "encoding/binary" - "encoding/json" "errors" "fmt" "math" "math/rand" "os" - "reflect" "sort" "strconv" "strings" @@ -45,6 +43,8 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" + storetesting "github.com/algorand/go-algorand/ledger/store/testing" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -97,17 +97,17 @@ func checkAccounts(t *testing.T, tx *sql.Tx, rnd basics.Round, accts map[basics. require.NoError(t, err) require.Equal(t, r, rnd) - aq, err := accountsInitDbQueries(tx) + aq, err := store.AccountsInitDbQueries(tx) require.NoError(t, err) - defer aq.close() + defer aq.Close() var totalOnline, totalOffline, totalNotPart uint64 for addr, data := range accts { expected := ledgercore.ToAccountData(data) - pad, err := aq.lookup(addr) + pad, err := aq.LookupAccount(addr) require.NoError(t, err) - d := pad.accountData.GetLedgerCoreAccountData() + d := pad.AccountData.GetLedgerCoreAccountData() require.Equal(t, expected, d) switch d.Status { @@ -134,10 +134,10 @@ func checkAccounts(t *testing.T, tx *sql.Tx, rnd basics.Round, accts map[basics. require.Equal(t, totalOnline+totalOffline, totals.Participating().Raw) require.Equal(t, totalOnline+totalOffline+totalNotPart, totals.All().Raw) - d, err := aq.lookup(ledgertesting.RandomAddress()) + d, err := aq.LookupAccount(ledgertesting.RandomAddress()) require.NoError(t, err) - require.Equal(t, rnd, d.round) - require.Equal(t, d.accountData, baseAccountData{}) + require.Equal(t, rnd, d.Round) + require.Equal(t, d.AccountData, store.BaseAccountData{}) proto := config.Consensus[protocol.ConsensusCurrentVersion] @@ -302,7 +302,7 @@ func TestAccountDBRound(t *testing.T) { knownAddresses := make(map[basics.Address]int64) for _, delta := range updatesCnt.deltas { - knownAddresses[delta.oldAcct.addr] = delta.oldAcct.rowid + knownAddresses[delta.oldAcct.Addr] = delta.oldAcct.Rowid } err = resourceUpdatesCnt.resourcesLoadOld(tx, knownAddresses) @@ -428,14 +428,14 @@ func TestAccountDBInMemoryAcct(t *testing.T) { outAccountDeltas := makeCompactAccountDeltas(accountDeltas, basics.Round(1), true, baseAccounts) require.Equal(t, 1, len(outAccountDeltas.deltas)) - require.Equal(t, accountDelta{newAcct: baseAccountData{UpdateRound: lastRound}, nAcctDeltas: numAcctDeltas, address: addr}, outAccountDeltas.deltas[0]) + require.Equal(t, accountDelta{newAcct: store.BaseAccountData{UpdateRound: lastRound}, nAcctDeltas: numAcctDeltas, address: addr}, outAccountDeltas.deltas[0]) require.Equal(t, 1, len(outAccountDeltas.misses)) outResourcesDeltas := makeCompactResourceDeltas(accountDeltas, basics.Round(1), true, baseAccounts, baseResources) require.Equal(t, 1, len(outResourcesDeltas.deltas)) require.Equal(t, resourceDelta{ - oldResource: persistedResourcesData{aidx: 100}, newResource: makeResourcesData(lastRound - 1), + oldResource: store.PersistedResourcesData{Aidx: 100}, newResource: store.MakeResourcesData(lastRound - 1), nAcctDeltas: numResDeltas, address: addr, }, outResourcesDeltas.deltas[0], @@ -447,7 +447,7 @@ func TestAccountDBInMemoryAcct(t *testing.T) { knownAddresses := make(map[basics.Address]int64) for _, delta := range outAccountDeltas.deltas { - knownAddresses[delta.oldAcct.addr] = delta.oldAcct.rowid + knownAddresses[delta.oldAcct.Addr] = delta.oldAcct.Rowid } err = outResourcesDeltas.resourcesLoadOld(tx, knownAddresses) @@ -457,13 +457,13 @@ func TestAccountDBInMemoryAcct(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(updatedAccts)) // we store empty even for deleted accounts require.Equal(t, - persistedAccountData{addr: addr, round: basics.Round(lastRound)}, + store.PersistedAccountData{Addr: addr, Round: basics.Round(lastRound)}, updatedAccts[0], ) require.Equal(t, 1, len(updatesResources[addr])) // we store empty even for deleted resources require.Equal(t, - persistedResourcesData{addrid: 0, aidx: 100, data: makeResourcesData(0), round: basics.Round(lastRound)}, + store.PersistedResourcesData{Addrid: 0, Aidx: 100, Data: store.MakeResourcesData(0), Round: basics.Round(lastRound)}, updatesResources[addr][0], ) @@ -726,9 +726,9 @@ func benchmarkReadingRandomBalances(b *testing.B, inMemory bool) { accounts := benchmarkInitBalances(b, b.N, dbs, protocol.ConsensusCurrentVersion) - qs, err := accountsInitDbQueries(dbs.Rdb.Handle) + qs, err := store.AccountsInitDbQueries(dbs.Rdb.Handle) require.NoError(b, err) - defer qs.close() + defer qs.Close() // read all the balances in the database, shuffled addrs := make([]basics.Address, len(accounts)) @@ -742,7 +742,7 @@ func benchmarkReadingRandomBalances(b *testing.B, inMemory bool) { // only measure the actual fetch time b.ResetTimer() for _, addr := range addrs { - _, err = qs.lookup(addr) + _, err = qs.LookupAccount(addr) require.NoError(b, err) } } @@ -969,8 +969,8 @@ func TestAccountsReencoding(t *testing.T) { func TestAccountsDbQueriesCreateClose(t *testing.T) { partitiontest.PartitionTest(t) - dbs, _ := dbOpenTest(t, true) - setDbLogging(t, dbs) + dbs, _ := storetesting.DbOpenTest(t, true) + storetesting.SetDbLogging(t, dbs) defer dbs.Close() err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { @@ -978,13 +978,14 @@ func TestAccountsDbQueriesCreateClose(t *testing.T) { return nil }) require.NoError(t, err) - qs, err := accountsInitDbQueries(dbs.Rdb.Handle) + qs, err := store.AccountsInitDbQueries(dbs.Rdb.Handle) require.NoError(t, err) - require.NotNil(t, qs.listCreatablesStmt) - qs.close() - require.Nil(t, qs.listCreatablesStmt) - qs.close() - require.Nil(t, qs.listCreatablesStmt) + // TODO[store-refactor]: internals are opaque, once we move the the remainder of accountdb we can mvoe this too + // require.NotNil(t, qs.listCreatablesStmt) + qs.Close() + // require.Nil(t, qs.listCreatablesStmt) + qs.Close() + // require.Nil(t, qs.listCreatablesStmt) } func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder bool) { @@ -1031,7 +1032,7 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder boo chunk.Balances = make([]encodedBalanceRecordV6, chunkSize) for i := uint64(0); i < chunkSize; i++ { var randomAccount encodedBalanceRecordV6 - accountData := baseAccountData{RewardsBase: accountsLoaded + i} + accountData := store.BaseAccountData{RewardsBase: accountsLoaded + i} accountData.MicroAlgos.Raw = crypto.RandUint63() randomAccount.AccountData = protocol.Encode(&accountData) crypto.RandBytes(randomAccount.Address[:]) @@ -1081,37 +1082,6 @@ func BenchmarkWriteCatchpointStagingBalances(b *testing.B) { } } -func TestKeyPrefixIntervalPreprocessing(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - testCases := []struct { - input []byte - outputPrefix []byte - outputPrefixIncr []byte - }{ - {input: []byte{0xAB, 0xCD}, outputPrefix: []byte{0xAB, 0xCD}, outputPrefixIncr: []byte{0xAB, 0xCE}}, - {input: []byte{0xFF}, outputPrefix: []byte{0xFF}, outputPrefixIncr: nil}, - {input: []byte{0xFE, 0xFF}, outputPrefix: []byte{0xFE, 0xFF}, outputPrefixIncr: []byte{0xFF}}, - {input: []byte{0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFF}, outputPrefixIncr: nil}, - {input: []byte{0xAB, 0xCD}, outputPrefix: []byte{0xAB, 0xCD}, outputPrefixIncr: []byte{0xAB, 0xCE}}, - {input: []byte{0x1E, 0xFF, 0xFF}, outputPrefix: []byte{0x1E, 0xFF, 0xFF}, outputPrefixIncr: []byte{0x1F}}, - {input: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefixIncr: []byte{0xFF, 0xFF}}, - {input: []byte{0x00, 0xFF}, outputPrefix: []byte{0x00, 0xFF}, outputPrefixIncr: []byte{0x01}}, - {input: []byte(string("bx:123")), outputPrefix: []byte(string("bx:123")), outputPrefixIncr: []byte(string("bx:124"))}, - {input: []byte{}, outputPrefix: []byte{}, outputPrefixIncr: nil}, - {input: nil, outputPrefix: []byte{}, outputPrefixIncr: nil}, - {input: []byte{0x1E, 0xFF, 0xFF}, outputPrefix: []byte{0x1E, 0xFF, 0xFF}, outputPrefixIncr: []byte{0x1F}}, - {input: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefixIncr: []byte{0xFF, 0xFF}}, - {input: []byte{0x00, 0xFF}, outputPrefix: []byte{0x00, 0xFF}, outputPrefixIncr: []byte{0x01}}, - } - for _, tc := range testCases { - actualOutputPrefix, actualOutputPrefixIncr := keyPrefixIntervalPreprocessing(tc.input) - require.Equal(t, tc.outputPrefix, actualOutputPrefix) - require.Equal(t, tc.outputPrefixIncr, actualOutputPrefixIncr) - } -} - func TestLookupKeysByPrefix(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() @@ -1123,9 +1093,9 @@ func TestLookupKeysByPrefix(t *testing.T) { // return account data, initialize DB tables from accountsInitTest _ = benchmarkInitBalances(t, 1, dbs, protocol.ConsensusCurrentVersion) - qs, err := accountsInitDbQueries(dbs.Rdb.Handle) + qs, err := store.AccountsInitDbQueries(dbs.Rdb.Handle) require.NoError(t, err) - defer qs.close() + defer qs.Close() kvPairDBPrepareSet := []struct { key []byte @@ -1154,19 +1124,19 @@ func TestLookupKeysByPrefix(t *testing.T) { require.NoError(t, err) // writer is only for kvstore - writer, err := makeAccountsSQLWriter(tx, true, true, true, true) + writer, err := store.MakeAccountsSQLWriter(tx, true, true, true, true) if err != nil { return } for i := 0; i < len(kvPairDBPrepareSet); i++ { - err := writer.upsertKvPair(string(kvPairDBPrepareSet[i].key), kvPairDBPrepareSet[i].value) + err := writer.UpsertKvPair(string(kvPairDBPrepareSet[i].key), kvPairDBPrepareSet[i].value) require.NoError(t, err) } err = tx.Commit() require.NoError(t, err) - writer.close() + writer.Close() testCases := []struct { prefix []byte @@ -1278,7 +1248,7 @@ func TestLookupKeysByPrefix(t *testing.T) { for index, testCase := range testCases { t.Run("lookupKVByPrefix-testcase-"+strconv.Itoa(index), func(t *testing.T) { actual := make(map[string]bool) - _, err := qs.lookupKeysByPrefix(string(testCase.prefix), uint64(len(kvPairDBPrepareSet)), actual, 0) + _, err := qs.LookupKeysByPrefix(string(testCase.prefix), uint64(len(kvPairDBPrepareSet)), actual, 0) if err != nil { require.NotEmpty(t, testCase.err, testCase.prefix) require.Contains(t, err.Error(), testCase.err) @@ -1304,9 +1274,9 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { // return account data, initialize DB tables from accountsInitTest _ = benchmarkInitBalances(b, 1, dbs, protocol.ConsensusCurrentVersion) - qs, err := accountsInitDbQueries(dbs.Rdb.Handle) + qs, err := store.AccountsInitDbQueries(dbs.Rdb.Handle) require.NoError(b, err) - defer qs.close() + defer qs.Close() currentDBSize := 0 nextDBSize := 2 @@ -1322,7 +1292,7 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { require.NoError(b, err) // writer is only for kvstore - writer, err := makeAccountsSQLWriter(tx, true, true, true, true) + writer, err := store.MakeAccountsSQLWriter(tx, true, true, true, true) if err != nil { return } @@ -1334,7 +1304,7 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { crypto.RandBytes(valueBuffer) appID := basics.AppIndex(crypto.RandUint64()) boxKey := logic.MakeBoxKey(appID, string(nameBuffer)) - err = writer.upsertKvPair(boxKey, valueBuffer) + err = writer.UpsertKvPair(boxKey, valueBuffer) require.NoError(b, err) if i == 0 { @@ -1343,7 +1313,7 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { } err = tx.Commit() require.NoError(b, err) - writer.close() + writer.Close() // benchmark the query against large DB, see if we have O(log N) speed currentDBSize = nextDBSize @@ -1352,7 +1322,7 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { b.Run("lookupKVByPrefix-DBsize"+strconv.Itoa(currentDBSize), func(b *testing.B) { for i := 0; i < b.N; i++ { results := make(map[string]bool) - _, err := qs.lookupKeysByPrefix(prefix, uint64(currentDBSize), results, 0) + _, err := qs.LookupKeysByPrefix(prefix, uint64(currentDBSize), results, 0) require.NoError(b, err) require.True(b, len(results) >= 1) } @@ -1362,7 +1332,7 @@ func BenchmarkLookupKeyByPrefix(b *testing.B) { // upsert updates existing or inserts a new entry func (a *compactResourcesDeltas) upsert(delta resourceDelta) { - if idx, exist := a.cache[accountCreatable{address: delta.address, index: delta.oldResource.aidx}]; exist { + if idx, exist := a.cache[accountCreatable{address: delta.address, index: delta.oldResource.Aidx}]; exist { a.deltas[idx] = delta return } @@ -1370,13 +1340,13 @@ func (a *compactResourcesDeltas) upsert(delta resourceDelta) { } // upsertOld updates existing or inserts a new partial entry with only old field filled -func (a *compactAccountDeltas) upsertOld(old persistedAccountData) { - addr := old.addr +func (a *compactAccountDeltas) upsertOld(old store.PersistedAccountData) { + addr := old.Addr if idx, exist := a.cache[addr]; exist { a.deltas[idx].oldAcct = old return } - a.insert(accountDelta{oldAcct: old, address: old.addr}) + a.insert(accountDelta{oldAcct: old, address: old.Addr}) } // upsert updates existing or inserts a new entry @@ -1405,7 +1375,7 @@ func TestCompactAccountDeltas(t *testing.T) { a.Zero(ad.len()) a.Panics(func() { ad.getByIdx(0) }) - sample1 := accountDelta{newAcct: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 123}}, address: addr} + sample1 := accountDelta{newAcct: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 123}}, address: addr} ad.upsert(addr, sample1) data, idx = ad.get(addr) a.NotEqual(-1, idx) @@ -1416,7 +1386,7 @@ func TestCompactAccountDeltas(t *testing.T) { a.Equal(addr, data.address) a.Equal(sample1, data) - sample2 := accountDelta{newAcct: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 456}}, address: addr} + sample2 := accountDelta{newAcct: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 456}}, address: addr} ad.upsert(addr, sample2) data, idx = ad.get(addr) a.NotEqual(-1, idx) @@ -1437,7 +1407,7 @@ func TestCompactAccountDeltas(t *testing.T) { a.Equal(addr, data.address) a.Equal(sample2, data) - old1 := persistedAccountData{addr: addr, accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 789}}} + old1 := store.PersistedAccountData{Addr: addr, AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 789}}} ad.upsertOld(old1) a.Equal(1, ad.len()) data = ad.getByIdx(0) @@ -1445,7 +1415,7 @@ func TestCompactAccountDeltas(t *testing.T) { a.Equal(accountDelta{newAcct: sample2.newAcct, oldAcct: old1, address: addr}, data) addr1 := ledgertesting.RandomAddress() - old2 := persistedAccountData{addr: addr1, accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 789}}} + old2 := store.PersistedAccountData{Addr: addr1, AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 789}}} ad.upsertOld(old2) a.Equal(2, ad.len()) data = ad.getByIdx(0) @@ -1453,7 +1423,7 @@ func TestCompactAccountDeltas(t *testing.T) { a.Equal(accountDelta{newAcct: sample2.newAcct, oldAcct: old1, address: addr}, data) data = ad.getByIdx(1) - a.Equal(addr1, data.oldAcct.addr) + a.Equal(addr1, data.oldAcct.Addr) a.Equal(accountDelta{oldAcct: old2, address: addr1}, data) // apply old on empty delta object, expect no changes @@ -1474,14 +1444,15 @@ func TestCompactAccountDeltas(t *testing.T) { } // upsertOld updates existing or inserts a new partial entry with only old field filled -func (a *compactResourcesDeltas) upsertOld(addr basics.Address, old persistedResourcesData) { - if idx, exist := a.cache[accountCreatable{address: addr, index: old.aidx}]; exist { +func (a *compactResourcesDeltas) upsertOld(addr basics.Address, old store.PersistedResourcesData) { + if idx, exist := a.cache[accountCreatable{address: addr, index: old.Aidx}]; exist { a.deltas[idx].oldResource = old return } idx := a.insert(resourceDelta{oldResource: old, address: addr}) a.deltas[idx].address = addr } + func TestCompactResourceDeltas(t *testing.T) { partitiontest.PartitionTest(t) @@ -1500,7 +1471,7 @@ func TestCompactResourceDeltas(t *testing.T) { a.Zero(ad.len()) a.Panics(func() { ad.getByIdx(0) }) - sample1 := resourceDelta{newResource: resourcesData{Total: 123}, address: addr, oldResource: persistedResourcesData{aidx: 1}} + sample1 := resourceDelta{newResource: store.ResourcesData{Total: 123}, address: addr, oldResource: store.PersistedResourcesData{Aidx: 1}} ad.upsert(sample1) data, idx = ad.get(addr, 1) a.NotEqual(-1, idx) @@ -1511,7 +1482,7 @@ func TestCompactResourceDeltas(t *testing.T) { a.Equal(addr, data.address) a.Equal(sample1, data) - sample2 := resourceDelta{newResource: resourcesData{Total: 456}, address: addr, oldResource: persistedResourcesData{aidx: 1}} + sample2 := resourceDelta{newResource: store.ResourcesData{Total: 456}, address: addr, oldResource: store.PersistedResourcesData{Aidx: 1}} ad.upsert(sample2) data, idx = ad.get(addr, 1) a.NotEqual(-1, idx) @@ -1532,7 +1503,7 @@ func TestCompactResourceDeltas(t *testing.T) { a.Equal(addr, data.address) a.Equal(sample2, data) - old1 := persistedResourcesData{addrid: 111, aidx: 1, data: resourcesData{Total: 789}} + old1 := store.PersistedResourcesData{Addrid: 111, Aidx: 1, Data: store.ResourcesData{Total: 789}} ad.upsertOld(addr, old1) a.Equal(1, ad.len()) data = ad.getByIdx(0) @@ -1540,7 +1511,7 @@ func TestCompactResourceDeltas(t *testing.T) { a.Equal(resourceDelta{newResource: sample2.newResource, oldResource: old1, address: addr}, data) addr1 := ledgertesting.RandomAddress() - old2 := persistedResourcesData{addrid: 222, aidx: 2, data: resourcesData{Total: 789}} + old2 := store.PersistedResourcesData{Addrid: 222, Aidx: 2, Data: store.ResourcesData{Total: 789}} ad.upsertOld(addr1, old2) a.Equal(2, ad.len()) data = ad.getByIdx(0) @@ -1558,7 +1529,7 @@ func TestCompactResourceDeltas(t *testing.T) { a.Equal(resourceDelta{newResource: sample2.newResource, oldResource: old2, address: addr}, data) addr2 := ledgertesting.RandomAddress() - sample2.oldResource.aidx = 2 + sample2.oldResource.Aidx = 2 sample2.address = addr2 idx = ad.insert(sample2) a.Equal(3, ad.len()) @@ -1571,1226 +1542,6 @@ func TestCompactResourceDeltas(t *testing.T) { a.Equal(sample2, data) } -func TestResourcesDataApp(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - a := require.New(t) - - rd := resourcesData{} - a.False(rd.IsApp()) - a.True(rd.IsEmpty()) - - rd = makeResourcesData(1) - a.False(rd.IsApp()) - a.False(rd.IsHolding()) - a.False(rd.IsOwning()) - a.True(rd.IsEmpty()) - - // check empty - appParamsEmpty := basics.AppParams{} - rd = resourcesData{} - rd.SetAppParams(appParamsEmpty, false) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParamsEmpty, rd.GetAppParams()) - - appLocalEmpty := basics.AppLocalState{} - rd = resourcesData{} - rd.SetAppLocalState(appLocalEmpty) - a.True(rd.IsApp()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - // check both empty - rd = resourcesData{} - rd.SetAppLocalState(appLocalEmpty) - rd.SetAppParams(appParamsEmpty, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParamsEmpty, rd.GetAppParams()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - // Since some steps use randomly generated input, the test is run N times - // to cover a larger search space of inputs. - for i := 0; i < 1000; i++ { - // check empty states + non-empty params - appParams := ledgertesting.RandomAppParams() - rd = resourcesData{} - rd.SetAppLocalState(appLocalEmpty) - rd.SetAppParams(appParams, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParams, rd.GetAppParams()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - appState := ledgertesting.RandomAppLocalState() - rd.SetAppLocalState(appState) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParams, rd.GetAppParams()) - a.Equal(appState, rd.GetAppLocalState()) - - // check ClearAppLocalState - rd.ClearAppLocalState() - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParams, rd.GetAppParams()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - // check ClearAppParams - rd.SetAppLocalState(appState) - rd.ClearAppParams() - a.True(rd.IsApp()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - if appState.Schema.NumEntries() == 0 { - a.True(rd.IsEmptyAppFields()) - } else { - a.False(rd.IsEmptyAppFields()) - } - a.False(rd.IsEmpty()) - a.Equal(appParamsEmpty, rd.GetAppParams()) - a.Equal(appState, rd.GetAppLocalState()) - - // check both clear - rd.ClearAppLocalState() - a.False(rd.IsApp()) - a.False(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.True(rd.IsEmpty()) - a.Equal(appParamsEmpty, rd.GetAppParams()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - // check params clear when non-empty params and empty holding - rd = resourcesData{} - rd.SetAppLocalState(appLocalEmpty) - rd.SetAppParams(appParams, true) - rd.ClearAppParams() - a.True(rd.IsApp()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - a.Equal(appParamsEmpty, rd.GetAppParams()) - a.Equal(appLocalEmpty, rd.GetAppLocalState()) - - rd = resourcesData{} - rd.SetAppLocalState(appLocalEmpty) - a.True(rd.IsEmptyAppFields()) - a.True(rd.IsApp()) - a.False(rd.IsEmpty()) - a.Equal(rd.ResourceFlags, resourceFlagsEmptyApp) - rd.ClearAppLocalState() - a.False(rd.IsApp()) - a.True(rd.IsEmptyAppFields()) - a.True(rd.IsEmpty()) - a.Equal(rd.ResourceFlags, resourceFlagsNotHolding) - - // check migration flow (accountDataResources) - // 1. both exist and empty - rd = makeResourcesData(0) - rd.SetAppLocalState(appLocalEmpty) - rd.SetAppParams(appParamsEmpty, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - - // 2. both exist and not empty - rd = makeResourcesData(0) - rd.SetAppLocalState(appState) - rd.SetAppParams(appParams, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - - // 3. both exist: holding not empty, param is empty - rd = makeResourcesData(0) - rd.SetAppLocalState(appState) - rd.SetAppParams(appParamsEmpty, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - if appState.Schema.NumEntries() == 0 { - a.True(rd.IsEmptyAppFields()) - } else { - a.False(rd.IsEmptyAppFields()) - } - a.False(rd.IsEmpty()) - - // 4. both exist: holding empty, param is not empty - rd = makeResourcesData(0) - rd.SetAppLocalState(appLocalEmpty) - rd.SetAppParams(appParams, true) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - - // 5. holding does not exist and params is empty - rd = makeResourcesData(0) - rd.SetAppParams(appParamsEmpty, false) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - - // 6. holding does not exist and params is not empty - rd = makeResourcesData(0) - rd.SetAppParams(appParams, false) - a.True(rd.IsApp()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.False(rd.IsEmptyAppFields()) - a.False(rd.IsEmpty()) - - // 7. holding exist and not empty and params does not exist - rd = makeResourcesData(0) - rd.SetAppLocalState(appState) - a.True(rd.IsApp()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - if appState.Schema.NumEntries() == 0 { - a.True(rd.IsEmptyAppFields()) - } else { - a.False(rd.IsEmptyAppFields()) - } - a.False(rd.IsEmpty()) - - // 8. both do not exist - rd = makeResourcesData(0) - a.False(rd.IsApp()) - a.False(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAppFields()) - a.True(rd.IsEmpty()) - } -} - -func TestResourcesDataAsset(t *testing.T) { - partitiontest.PartitionTest(t) - - a := require.New(t) - - rd := resourcesData{} - a.False(rd.IsAsset()) - a.True(rd.IsEmpty()) - - rd = makeResourcesData(1) - a.False(rd.IsAsset()) - a.False(rd.IsHolding()) - a.False(rd.IsOwning()) - a.True(rd.IsEmpty()) - - // check empty - assetParamsEmpty := basics.AssetParams{} - rd = resourcesData{} - rd.SetAssetParams(assetParamsEmpty, false) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParamsEmpty, rd.GetAssetParams()) - - assetHoldingEmpty := basics.AssetHolding{} - rd = resourcesData{} - rd.SetAssetHolding(assetHoldingEmpty) - a.True(rd.IsAsset()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - // check both empty - rd = resourcesData{} - rd.SetAssetHolding(assetHoldingEmpty) - rd.SetAssetParams(assetParamsEmpty, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParamsEmpty, rd.GetAssetParams()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - // check empty states + non-empty params - assetParams := ledgertesting.RandomAssetParams() - rd = resourcesData{} - rd.SetAssetHolding(assetHoldingEmpty) - rd.SetAssetParams(assetParams, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParams, rd.GetAssetParams()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - assetHolding := ledgertesting.RandomAssetHolding(true) - rd.SetAssetHolding(assetHolding) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParams, rd.GetAssetParams()) - a.Equal(assetHolding, rd.GetAssetHolding()) - - // check ClearAssetHolding - rd.ClearAssetHolding() - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParams, rd.GetAssetParams()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - // check ClearAssetParams - rd.SetAssetHolding(assetHolding) - rd.ClearAssetParams() - a.True(rd.IsAsset()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParamsEmpty, rd.GetAssetParams()) - a.Equal(assetHolding, rd.GetAssetHolding()) - - // check both clear - rd.ClearAssetHolding() - a.False(rd.IsAsset()) - a.False(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.True(rd.IsEmpty()) - a.Equal(assetParamsEmpty, rd.GetAssetParams()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - // check params clear when non-empty params and empty holding - rd = resourcesData{} - rd.SetAssetHolding(assetHoldingEmpty) - rd.SetAssetParams(assetParams, true) - rd.ClearAssetParams() - a.True(rd.IsAsset()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - a.Equal(assetParamsEmpty, rd.GetAssetParams()) - a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) - - rd = resourcesData{} - rd.SetAssetHolding(assetHoldingEmpty) - a.True(rd.IsEmptyAssetFields()) - a.True(rd.IsAsset()) - a.False(rd.IsEmpty()) - a.Equal(rd.ResourceFlags, resourceFlagsEmptyAsset) - rd.ClearAssetHolding() - a.False(rd.IsAsset()) - a.True(rd.IsEmptyAssetFields()) - a.True(rd.IsEmpty()) - a.Equal(rd.ResourceFlags, resourceFlagsNotHolding) - - // check migration operations (accountDataResources) - // 1. both exist and empty - rd = makeResourcesData(0) - rd.SetAssetHolding(assetHoldingEmpty) - rd.SetAssetParams(assetParamsEmpty, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 2. both exist and not empty - rd = makeResourcesData(0) - rd.SetAssetHolding(assetHolding) - rd.SetAssetParams(assetParams, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 3. both exist: holding not empty, param is empty - rd = makeResourcesData(0) - rd.SetAssetHolding(assetHolding) - rd.SetAssetParams(assetParamsEmpty, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 4. both exist: holding empty, param is not empty - rd = makeResourcesData(0) - rd.SetAssetHolding(assetHoldingEmpty) - rd.SetAssetParams(assetParams, true) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 5. holding does not exist and params is empty - rd = makeResourcesData(0) - rd.SetAssetParams(assetParamsEmpty, false) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 6. holding does not exist and params is not empty - rd = makeResourcesData(0) - rd.SetAssetParams(assetParams, false) - a.True(rd.IsAsset()) - a.True(rd.IsOwning()) - a.False(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 7. holding exist and not empty and params does not exist - rd = makeResourcesData(0) - rd.SetAssetHolding(assetHolding) - a.True(rd.IsAsset()) - a.False(rd.IsOwning()) - a.True(rd.IsHolding()) - a.False(rd.IsEmptyAssetFields()) - a.False(rd.IsEmpty()) - - // 8. both do not exist - rd = makeResourcesData(0) - a.False(rd.IsAsset()) - a.False(rd.IsOwning()) - a.False(rd.IsHolding()) - a.True(rd.IsEmptyAssetFields()) - a.True(rd.IsEmpty()) -} - -// TestResourcesDataSetData checks combinations of old/new values when -// updating resourceData from resourceDelta -func TestResourcesDataSetData(t *testing.T) { - partitiontest.PartitionTest(t) - - a := require.New(t) - - type deltaCode int - const ( - tri deltaCode = iota + 1 - del - emp - act - ) - - // apply deltas encoded as deltaCode to a base resourcesData for both apps and assets - apply := func(t *testing.T, base resourcesData, testType basics.CreatableType, pcode, hcode deltaCode) resourcesData { - if testType == basics.AssetCreatable { - var p ledgercore.AssetParamsDelta - var h ledgercore.AssetHoldingDelta - switch pcode { - case tri: - break - case del: - p = ledgercore.AssetParamsDelta{Deleted: true} - case emp: - p = ledgercore.AssetParamsDelta{Params: &basics.AssetParams{}} - case act: - p = ledgercore.AssetParamsDelta{Params: &basics.AssetParams{Total: 1000}} - default: - t.Logf("invalid pcode: %d", pcode) - t.Fail() - } - switch hcode { - case tri: - break - case del: - h = ledgercore.AssetHoldingDelta{Deleted: true} - case emp: - h = ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{}} - case act: - h = ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{Amount: 555}} - default: - t.Logf("invalid hcode: %d", hcode) - t.Fail() - } - base.SetAssetData(p, h) - } else { - var p ledgercore.AppParamsDelta - var h ledgercore.AppLocalStateDelta - switch pcode { - case tri: - break - case del: - p = ledgercore.AppParamsDelta{Deleted: true} - case emp: - p = ledgercore.AppParamsDelta{Params: &basics.AppParams{}} - case act: - p = ledgercore.AppParamsDelta{Params: &basics.AppParams{ClearStateProgram: []byte{4, 5, 6}}} - default: - t.Logf("invalid pcode: %d", pcode) - t.Fail() - } - switch hcode { - case tri: - break - case del: - h = ledgercore.AppLocalStateDelta{Deleted: true} - case emp: - h = ledgercore.AppLocalStateDelta{LocalState: &basics.AppLocalState{}} - case act: - h = ledgercore.AppLocalStateDelta{LocalState: &basics.AppLocalState{Schema: basics.StateSchema{NumByteSlice: 5}}} - default: - t.Logf("invalid hcode: %d", hcode) - t.Fail() - } - base.SetAppData(p, h) - } - - return base - } - - itb := func(i int) (b bool) { - return i != 0 - } - - type testcase struct { - p deltaCode - h deltaCode - isAsset int - isOwning int - isHolding int - isEmptyFields int - isEmpty int - } - - empty := func(testType basics.CreatableType) resourcesData { - return makeResourcesData(0) - } - emptyParamsNoHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetParams(basics.AssetParams{}, false) - } else { - rd.SetAppParams(basics.AppParams{}, false) - } - return rd - } - emptyParamsEmptyHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{}) - rd.SetAssetParams(basics.AssetParams{}, true) - } else { - rd.SetAppLocalState(basics.AppLocalState{}) - rd.SetAppParams(basics.AppParams{}, true) - } - return rd - } - emptyParamsNotEmptyHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) - rd.SetAssetParams(basics.AssetParams{}, true) - } else { - rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) - rd.SetAppParams(basics.AppParams{}, true) - } - return rd - } - paramsNoHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetParams(basics.AssetParams{Total: 222}, false) - } else { - rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, false) - } - return rd - } - paramsEmptyHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{}) - rd.SetAssetParams(basics.AssetParams{Total: 222}, true) - } else { - rd.SetAppLocalState(basics.AppLocalState{}) - rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, true) - } - return rd - } - paramsAndHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) - rd.SetAssetParams(basics.AssetParams{Total: 222}, true) - } else { - rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) - rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, true) - } - return rd - } - noParamsEmptyHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{}) - } else { - rd.SetAppLocalState(basics.AppLocalState{}) - } - return rd - } - noParamsNotEmptyHolding := func(testType basics.CreatableType) resourcesData { - rd := makeResourcesData(0) - if testType == basics.AssetCreatable { - rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) - } else { - rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) - } - return rd - } - - var tests = []struct { - name string - baseRD func(testType basics.CreatableType) resourcesData - testcases []testcase - }{ - { - "empty_base", empty, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 0, 0, 0, 1, 1}, - {del, tri, 0, 0, 0, 1, 1}, - {emp, tri, 1, 1, 0, 1, 0}, - {act, tri, 1, 1, 0, 0, 0}, - - {tri, del, 0, 0, 0, 1, 1}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 0, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 0, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - - { - "empty_params_no_holding", emptyParamsNoHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 0, 1, 0}, - {del, tri, 0, 0, 0, 1, 1}, - {emp, tri, 1, 1, 0, 1, 0}, - {act, tri, 1, 1, 0, 0, 0}, - - {tri, del, 1, 1, 0, 1, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "empty_params_empty_holding", emptyParamsEmptyHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 1, 1, 0}, - {del, tri, 1, 0, 1, 1, 0}, - {emp, tri, 1, 1, 1, 1, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 1, 1, 0, 1, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "empty_params_not_empty_holding", emptyParamsNotEmptyHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 1, 0, 0}, - {del, tri, 1, 0, 1, 0, 0}, - {emp, tri, 1, 1, 1, 0, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 1, 1, 0, 1, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "params_no_holding", paramsNoHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 0, 0, 0}, - {del, tri, 0, 0, 0, 1, 1}, - {emp, tri, 1, 1, 0, 1, 0}, - {act, tri, 1, 1, 0, 0, 0}, - - {tri, del, 1, 1, 0, 0, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 0, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "params_empty_holding", paramsEmptyHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 1, 0, 0}, - {del, tri, 1, 0, 1, 1, 0}, - {emp, tri, 1, 1, 1, 1, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 1, 1, 0, 0, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 0, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "params_and_holding", paramsAndHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 1, 1, 0, 0}, - {del, tri, 1, 0, 1, 0, 0}, - {emp, tri, 1, 1, 1, 0, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 1, 1, 0, 0, 0}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 1, 1, 0, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 1, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "no_params_empty_holding", noParamsEmptyHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 0, 1, 1, 0}, - {del, tri, 1, 0, 1, 1, 0}, - {emp, tri, 1, 1, 1, 1, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 0, 0, 0, 1, 1}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 0, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 0, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - { - "no_params_not_empty_holding", noParamsNotEmptyHolding, - []testcase{ - // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty - {tri, tri, 1, 0, 1, 0, 0}, - {del, tri, 1, 0, 1, 0, 0}, - {emp, tri, 1, 1, 1, 0, 0}, - {act, tri, 1, 1, 1, 0, 0}, - - {tri, del, 0, 0, 0, 1, 1}, - {del, del, 0, 0, 0, 1, 1}, - {emp, del, 1, 1, 0, 1, 0}, - {act, del, 1, 1, 0, 0, 0}, - - {tri, emp, 1, 0, 1, 1, 0}, - {del, emp, 1, 0, 1, 1, 0}, - {emp, emp, 1, 1, 1, 1, 0}, - {act, emp, 1, 1, 1, 0, 0}, - - {tri, act, 1, 0, 1, 0, 0}, - {del, act, 1, 0, 1, 0, 0}, - {emp, act, 1, 1, 1, 0, 0}, - {act, act, 1, 1, 1, 0, 0}, - }, - }, - } - for _, testType := range []basics.CreatableType{basics.AssetCreatable, basics.AppCreatable} { - for _, test := range tests { - var testTypeStr string - if testType == basics.AssetCreatable { - testTypeStr = "asset" - } else { - testTypeStr = "app" - } - t.Run(fmt.Sprintf("test_%s_%s", testTypeStr, test.name), func(t *testing.T) { - for i, ts := range test.testcases { - t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - rd := test.baseRD(testType) - rd = apply(t, rd, testType, ts.p, ts.h) - if testType == basics.AssetCreatable { - a.Equal(itb(ts.isAsset), rd.IsAsset()) - a.Equal(itb(ts.isEmptyFields), rd.IsEmptyAssetFields()) - a.False(rd.IsApp()) - a.True(rd.IsEmptyAppFields()) - } else { - a.Equal(itb(ts.isAsset), rd.IsApp()) - a.Equal(itb(ts.isEmptyFields), rd.IsEmptyAppFields()) - a.False(rd.IsAsset()) - a.True(rd.IsEmptyAssetFields()) - } - a.Equal(itb(ts.isOwning), rd.IsOwning()) - a.Equal(itb(ts.isHolding), rd.IsHolding()) - a.Equal(itb(ts.isEmpty), rd.IsEmpty()) - }) - } - }) - } - } -} - -// TestResourceDataRoundtripConversion ensures that basics.AppLocalState, basics.AppParams, -// basics.AssetHolding, and basics.AssetParams can be converted to resourcesData and back without -// losing any data. It uses reflection to be sure that no new fields are omitted. -// -// In other words, this test makes sure any new fields in basics.AppLocalState, basics.AppParams, -// basics.AssetHolding, or basics.AssetParam also get added to resourcesData. -func TestResourceDataRoundtripConversion(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - t.Run("basics.AppLocalState", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&basics.AppLocalState{}) - basicsAppLocalState := *randObj.(*basics.AppLocalState) - - var data resourcesData - data.SetAppLocalState(basicsAppLocalState) - roundTripAppLocalState := data.GetAppLocalState() - - require.Equal(t, basicsAppLocalState, roundTripAppLocalState) - } - }) - - t.Run("basics.AppParams", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&basics.AppParams{}) - basicsAppParams := *randObj.(*basics.AppParams) - - for _, haveHoldings := range []bool{true, false} { - var data resourcesData - data.SetAppParams(basicsAppParams, haveHoldings) - roundTripAppParams := data.GetAppParams() - - require.Equal(t, basicsAppParams, roundTripAppParams) - } - } - }) - - t.Run("basics.AssetHolding", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&basics.AssetHolding{}) - basicsAssetHolding := *randObj.(*basics.AssetHolding) - - var data resourcesData - data.SetAssetHolding(basicsAssetHolding) - roundTripAssetHolding := data.GetAssetHolding() - - require.Equal(t, basicsAssetHolding, roundTripAssetHolding) - } - }) - - t.Run("basics.AssetParams", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&basics.AssetParams{}) - basicsAssetParams := *randObj.(*basics.AssetParams) - - for _, haveHoldings := range []bool{true, false} { - var data resourcesData - data.SetAssetParams(basicsAssetParams, haveHoldings) - roundTripAssetParams := data.GetAssetParams() - - require.Equal(t, basicsAssetParams, roundTripAssetParams) - } - } - }) -} - -// TestBaseAccountDataRoundtripConversion ensures that baseAccountData can be converted to -// ledgercore.AccountData and basics.AccountData and back without losing any data. It uses -// reflection to be sure that no new fields are omitted. -// -// In other words, this test makes sure any new fields in baseAccountData also get added to -// ledgercore.AccountData and basics.AccountData. You should add a manual override in this test if -// the field really only belongs in baseAccountData. -func TestBaseAccountDataRoundtripConversion(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - t.Run("ledgercore.AccountData", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&baseAccountData{}) - baseAccount := *randObj.(*baseAccountData) - - ledgercoreAccount := baseAccount.GetLedgerCoreAccountData() - var roundTripAccount baseAccountData - roundTripAccount.SetCoreAccountData(&ledgercoreAccount) - - // Manually set UpdateRound, since it is lost in GetLedgerCoreAccountData - roundTripAccount.UpdateRound = baseAccount.UpdateRound - - require.Equal(t, baseAccount, roundTripAccount) - } - }) - - t.Run("basics.AccountData", func(t *testing.T) { - t.Parallel() - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&baseAccountData{}) - baseAccount := *randObj.(*baseAccountData) - - basicsAccount := baseAccount.GetAccountData() - var roundTripAccount baseAccountData - roundTripAccount.SetAccountData(&basicsAccount) - - // Manually set UpdateRound, since it is lost in GetAccountData - roundTripAccount.UpdateRound = baseAccount.UpdateRound - - // Manually set resources, since resource information is lost in GetAccountData - roundTripAccount.TotalAssetParams = baseAccount.TotalAssetParams - roundTripAccount.TotalAssets = baseAccount.TotalAssets - roundTripAccount.TotalAppLocalStates = baseAccount.TotalAppLocalStates - roundTripAccount.TotalAppParams = baseAccount.TotalAppParams - - require.Equal(t, baseAccount, roundTripAccount) - } - }) -} - -// TestBasicsAccountDataRoundtripConversion ensures that basics.AccountData can be converted to -// baseAccountData and back without losing any data. It uses reflection to be sure that this test is -// always up-to-date with new fields. -// -// In other words, this test makes sure any new fields in basics.AccountData also get added to -// baseAccountData. -func TestBasicsAccountDataRoundtripConversion(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&basics.AccountData{}) - basicsAccount := *randObj.(*basics.AccountData) - - var baseAccount baseAccountData - baseAccount.SetAccountData(&basicsAccount) - roundTripAccount := baseAccount.GetAccountData() - - // Manually set resources, since GetAccountData doesn't attempt to restore them - roundTripAccount.AssetParams = basicsAccount.AssetParams - roundTripAccount.Assets = basicsAccount.Assets - roundTripAccount.AppLocalStates = basicsAccount.AppLocalStates - roundTripAccount.AppParams = basicsAccount.AppParams - - require.Equal(t, basicsAccount, roundTripAccount) - require.Equal(t, uint64(len(roundTripAccount.AssetParams)), baseAccount.TotalAssetParams) - require.Equal(t, uint64(len(roundTripAccount.Assets)), baseAccount.TotalAssets) - require.Equal(t, uint64(len(roundTripAccount.AppLocalStates)), baseAccount.TotalAppLocalStates) - require.Equal(t, uint64(len(roundTripAccount.AppParams)), baseAccount.TotalAppParams) - } -} - -// TestLedgercoreAccountDataRoundtripConversion ensures that ledgercore.AccountData can be converted -// to baseAccountData and back without losing any data. It uses reflection to be sure that no new -// fields are omitted. -// -// In other words, this test makes sure any new fields in ledgercore.AccountData also get added to -// baseAccountData. -func TestLedgercoreAccountDataRoundtripConversion(t *testing.T) { - partitiontest.PartitionTest(t) - t.Parallel() - - for i := 0; i < 1000; i++ { - randObj, _ := protocol.RandomizeObject(&ledgercore.AccountData{}) - ledgercoreAccount := *randObj.(*ledgercore.AccountData) - - var baseAccount baseAccountData - baseAccount.SetCoreAccountData(&ledgercoreAccount) - roundTripAccount := baseAccount.GetLedgerCoreAccountData() - - require.Equal(t, ledgercoreAccount, roundTripAccount) - } -} - -func TestBaseAccountDataIsEmpty(t *testing.T) { - partitiontest.PartitionTest(t) - positiveTesting := func(t *testing.T) { - var ba baseAccountData - require.True(t, ba.IsEmpty()) - for i := 0; i < 20; i++ { - h := crypto.Hash([]byte{byte(i)}) - rnd := binary.BigEndian.Uint64(h[:]) - ba.UpdateRound = rnd - require.True(t, ba.IsEmpty()) - } - } - var empty baseAccountData - negativeTesting := func(t *testing.T) { - for i := 0; i < 10000; i++ { - randObj, _ := protocol.RandomizeObjectField(&baseAccountData{}) - ba := randObj.(*baseAccountData) - if *ba == empty || ba.UpdateRound != 0 { - continue - } - require.False(t, ba.IsEmpty(), "base account : %v", ba) - } - } - structureTesting := func(t *testing.T) { - encoding, err := json.Marshal(&empty) - zeros32 := "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0" - expectedEncoding := `{"Status":0,"MicroAlgos":{"Raw":0},"RewardsBase":0,"RewardedMicroAlgos":{"Raw":0},"AuthAddr":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ","TotalAppSchemaNumUint":0,"TotalAppSchemaNumByteSlice":0,"TotalExtraAppPages":0,"TotalAssetParams":0,"TotalAssets":0,"TotalAppParams":0,"TotalAppLocalStates":0,"TotalBoxes":0,"TotalBoxBytes":0,"VoteID":[` + zeros32 + `],"SelectionID":[` + zeros32 + `],"VoteFirstValid":0,"VoteLastValid":0,"VoteKeyDilution":0,"StateProofID":[` + zeros32 + `,` + zeros32 + `],"UpdateRound":0}` - require.NoError(t, err) - require.Equal(t, expectedEncoding, string(encoding)) - } - t.Run("Positive", positiveTesting) - t.Run("Negative", negativeTesting) - t.Run("Structure", structureTesting) - -} - -func TestBaseOnlineAccountDataIsEmpty(t *testing.T) { - partitiontest.PartitionTest(t) - - positiveTesting := func(t *testing.T) { - var ba baseOnlineAccountData - require.True(t, ba.IsEmpty()) - require.True(t, ba.IsVotingEmpty()) - ba.MicroAlgos.Raw = 100 - require.True(t, ba.IsVotingEmpty()) - ba.RewardsBase = 200 - require.True(t, ba.IsVotingEmpty()) - } - var empty baseOnlineAccountData - negativeTesting := func(t *testing.T) { - for i := 0; i < 10; i++ { - randObj, _ := protocol.RandomizeObjectField(&baseOnlineAccountData{}) - ba := randObj.(*baseOnlineAccountData) - if *ba == empty { - continue - } - require.False(t, ba.IsEmpty(), "base account : %v", ba) - break - } - { - var ba baseOnlineAccountData - ba.MicroAlgos.Raw = 100 - require.False(t, ba.IsEmpty()) - } - { - var ba baseOnlineAccountData - ba.RewardsBase = 200 - require.False(t, ba.IsEmpty()) - } - } - structureTesting := func(t *testing.T) { - encoding, err := json.Marshal(&empty) - zeros32 := "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0" - expectedEncoding := `{"VoteID":[` + zeros32 + `],"SelectionID":[` + zeros32 + `],"VoteFirstValid":0,"VoteLastValid":0,"VoteKeyDilution":0,"StateProofID":[` + zeros32 + `,` + zeros32 + `],"MicroAlgos":{"Raw":0},"RewardsBase":0}` - require.NoError(t, err) - require.Equal(t, expectedEncoding, string(encoding)) - } - t.Run("Positive", positiveTesting) - t.Run("Negative", negativeTesting) - t.Run("Structure", structureTesting) - -} - -func TestBaseOnlineAccountDataGettersSetters(t *testing.T) { - partitiontest.PartitionTest(t) - - proto := config.Consensus[protocol.ConsensusCurrentVersion] - addr := ledgertesting.RandomAddress() - data := ledgertesting.RandomAccountData(1) - data.Status = basics.Online - crypto.RandBytes(data.VoteID[:]) - crypto.RandBytes(data.SelectionID[:]) - crypto.RandBytes(data.StateProofID[:]) - data.VoteFirstValid = basics.Round(crypto.RandUint64()) - data.VoteLastValid = basics.Round(crypto.RandUint64()) // int64 is the max sqlite can store - data.VoteKeyDilution = crypto.RandUint64() - - var ba baseOnlineAccountData - ad := ledgercore.ToAccountData(data) - ba.SetCoreAccountData(&ad) - - require.Equal(t, data.MicroAlgos, ba.MicroAlgos) - require.Equal(t, data.RewardsBase, ba.RewardsBase) - require.Equal(t, data.VoteID, ba.VoteID) - require.Equal(t, data.SelectionID, ba.SelectionID) - require.Equal(t, data.VoteFirstValid, ba.VoteFirstValid) - require.Equal(t, data.VoteLastValid, ba.VoteLastValid) - require.Equal(t, data.VoteKeyDilution, ba.VoteKeyDilution) - require.Equal(t, data.StateProofID, ba.StateProofID) - - normBalance := basics.NormalizedOnlineAccountBalance(basics.Online, data.RewardsBase, data.MicroAlgos, proto) - require.Equal(t, normBalance, ba.NormalizedOnlineBalance(proto)) - oa := ba.GetOnlineAccount(addr, normBalance) - - require.Equal(t, addr, oa.Address) - require.Equal(t, ba.MicroAlgos, oa.MicroAlgos) - require.Equal(t, ba.RewardsBase, oa.RewardsBase) - require.Equal(t, normBalance, oa.NormalizedOnlineBalance) - require.Equal(t, ba.VoteFirstValid, oa.VoteFirstValid) - require.Equal(t, ba.VoteLastValid, oa.VoteLastValid) - require.Equal(t, ba.StateProofID, oa.StateProofID) - - rewardsLevel := uint64(1) - microAlgos, _, _ := basics.WithUpdatedRewards( - proto, basics.Online, oa.MicroAlgos, basics.MicroAlgos{}, ba.RewardsBase, rewardsLevel, - ) - oad := ba.GetOnlineAccountData(proto, rewardsLevel) - - require.Equal(t, microAlgos, oad.MicroAlgosWithRewards) - require.Equal(t, ba.VoteID, oad.VoteID) - require.Equal(t, ba.SelectionID, oad.SelectionID) - require.Equal(t, ba.StateProofID, oad.StateProofID) - require.Equal(t, ba.VoteFirstValid, oad.VoteFirstValid) - require.Equal(t, ba.VoteLastValid, oad.VoteLastValid) - require.Equal(t, ba.VoteKeyDilution, oad.VoteKeyDilution) -} - -func TestBaseVotingDataGettersSetters(t *testing.T) { - partitiontest.PartitionTest(t) - - data := ledgertesting.RandomAccountData(1) - data.Status = basics.Online - crypto.RandBytes(data.VoteID[:]) - crypto.RandBytes(data.SelectionID[:]) - crypto.RandBytes(data.StateProofID[:]) - data.VoteFirstValid = basics.Round(crypto.RandUint64()) - data.VoteLastValid = basics.Round(crypto.RandUint64()) // int64 is the max sqlite can store - data.VoteKeyDilution = crypto.RandUint64() - - var bv baseVotingData - require.True(t, bv.IsEmpty()) - - ad := ledgercore.ToAccountData(data) - bv.SetCoreAccountData(&ad) - - require.False(t, bv.IsEmpty()) - require.Equal(t, data.VoteID, bv.VoteID) - require.Equal(t, data.SelectionID, bv.SelectionID) - require.Equal(t, data.VoteFirstValid, bv.VoteFirstValid) - require.Equal(t, data.VoteLastValid, bv.VoteLastValid) - require.Equal(t, data.VoteKeyDilution, bv.VoteKeyDilution) - require.Equal(t, data.StateProofID, bv.StateProofID) -} - -func TestBaseOnlineAccountDataReflect(t *testing.T) { - partitiontest.PartitionTest(t) - - require.Equal(t, 4, reflect.TypeOf(baseOnlineAccountData{}).NumField(), "update all getters and setters for baseOnlineAccountData and change the field count") -} - -func TestBaseVotingDataReflect(t *testing.T) { - partitiontest.PartitionTest(t) - - require.Equal(t, 7, reflect.TypeOf(baseVotingData{}).NumField(), "update all getters and setters for baseVotingData and change the field count") -} - func TestLookupAccountAddressFromAddressID(t *testing.T) { partitiontest.PartitionTest(t) @@ -2928,7 +1679,7 @@ func (m *mockAccountWriter) setResource(addr basics.Address, cidx basics.Creatab return nil } -func (m *mockAccountWriter) lookup(addr basics.Address) (pad persistedAccountData, ok bool, err error) { +func (m *mockAccountWriter) Lookup(addr basics.Address) (pad store.PersistedAccountData, ok bool, err error) { rowid, ok := m.addresses[addr] if !ok { return @@ -2938,13 +1689,13 @@ func (m *mockAccountWriter) lookup(addr basics.Address) (pad persistedAccountDat err = fmt.Errorf("not found %s", addr.String()) return } - pad.accountData.SetCoreAccountData(&data) - pad.addr = addr - pad.rowid = rowid + pad.AccountData.SetCoreAccountData(&data) + pad.Addr = addr + pad.Rowid = rowid return } -func (m *mockAccountWriter) lookupResource(addr basics.Address, cidx basics.CreatableIndex) (prd persistedResourcesData, ok bool, err error) { +func (m *mockAccountWriter) LookupResource(addr basics.Address, cidx basics.CreatableIndex) (prd store.PersistedResourcesData, ok bool, err error) { rowid, ok := m.addresses[addr] if !ok { return @@ -2955,23 +1706,23 @@ func (m *mockAccountWriter) lookupResource(addr basics.Address, cidx basics.Crea return } if res.AppLocalState != nil { - prd.data.SetAppLocalState(*res.AppLocalState) + prd.Data.SetAppLocalState(*res.AppLocalState) } if res.AppParams != nil { - prd.data.SetAppParams(*res.AppParams, prd.data.IsHolding()) + prd.Data.SetAppParams(*res.AppParams, prd.Data.IsHolding()) } if res.AssetHolding != nil { - prd.data.SetAssetHolding(*res.AssetHolding) + prd.Data.SetAssetHolding(*res.AssetHolding) } if res.AssetParams != nil { - prd.data.SetAssetParams(*res.AssetParams, prd.data.IsHolding()) + prd.Data.SetAssetParams(*res.AssetParams, prd.Data.IsHolding()) } - prd.addrid = rowid - prd.aidx = cidx + prd.Addrid = rowid + prd.Aidx = cidx return } -func (m *mockAccountWriter) insertAccount(addr basics.Address, normBalance uint64, data baseAccountData) (rowid int64, err error) { +func (m *mockAccountWriter) InsertAccount(addr basics.Address, normBalance uint64, data store.BaseAccountData) (rowid int64, err error) { rowid, ok := m.addresses[addr] if ok { err = fmt.Errorf("insertAccount: addr %s, rowid %d: UNIQUE constraint failed", addr.String(), rowid) @@ -2984,7 +1735,7 @@ func (m *mockAccountWriter) insertAccount(addr basics.Address, normBalance uint6 return } -func (m *mockAccountWriter) deleteAccount(rowid int64) (rowsAffected int64, err error) { +func (m *mockAccountWriter) DeleteAccount(rowid int64) (rowsAffected int64, err error) { var addr basics.Address var ok bool if addr, ok = m.rowids[rowid]; !ok { @@ -2998,7 +1749,7 @@ func (m *mockAccountWriter) deleteAccount(rowid int64) (rowsAffected int64, err return 1, nil } -func (m *mockAccountWriter) updateAccount(rowid int64, normBalance uint64, data baseAccountData) (rowsAffected int64, err error) { +func (m *mockAccountWriter) UpdateAccount(rowid int64, normBalance uint64, data store.BaseAccountData) (rowsAffected int64, err error) { if _, ok := m.rowids[rowid]; !ok { return 0, fmt.Errorf("updateAccount: not found rowid %d", rowid) } @@ -3013,19 +1764,19 @@ func (m *mockAccountWriter) updateAccount(rowid int64, normBalance uint64, data return 1, nil } -func (m *mockAccountWriter) insertResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowid int64, err error) { +func (m *mockAccountWriter) InsertResource(addrid int64, aidx basics.CreatableIndex, data store.ResourcesData) (rowid int64, err error) { key := mockResourcesKey{addrid, aidx} if _, ok := m.resources[key]; ok { return 0, fmt.Errorf("insertResource: (%d, %d): UNIQUE constraint failed", addrid, aidx) } // use persistedResourcesData.AccountResource for conversion - prd := persistedResourcesData{data: data} + prd := store.PersistedResourcesData{Data: data} new := prd.AccountResource() m.resources[key] = new return 1, nil } -func (m *mockAccountWriter) deleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) { +func (m *mockAccountWriter) DeleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) { key := mockResourcesKey{addrid, aidx} if _, ok := m.resources[key]; !ok { return 0, nil @@ -3034,14 +1785,14 @@ func (m *mockAccountWriter) deleteResource(addrid int64, aidx basics.CreatableIn return 1, nil } -func (m *mockAccountWriter) updateResource(addrid int64, aidx basics.CreatableIndex, data resourcesData) (rowsAffected int64, err error) { +func (m *mockAccountWriter) UpdateResource(addrid int64, aidx basics.CreatableIndex, data store.ResourcesData) (rowsAffected int64, err error) { key := mockResourcesKey{addrid, aidx} old, ok := m.resources[key] if !ok { return 0, fmt.Errorf("updateResource: not found (%d, %d)", addrid, aidx) } // use persistedResourcesData.AccountResource for conversion - prd := persistedResourcesData{data: data} + prd := store.PersistedResourcesData{Data: data} new := prd.AccountResource() if new == old { return 0, nil @@ -3050,25 +1801,25 @@ func (m *mockAccountWriter) updateResource(addrid int64, aidx basics.CreatableIn return 1, nil } -func (m *mockAccountWriter) upsertKvPair(key string, value []byte) error { +func (m *mockAccountWriter) UpsertKvPair(key string, value []byte) error { m.kvStore[key] = value return nil } -func (m *mockAccountWriter) deleteKvPair(key string) error { +func (m *mockAccountWriter) DeleteKvPair(key string) error { delete(m.kvStore, key) return nil } -func (m *mockAccountWriter) insertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) { +func (m *mockAccountWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) { return 0, fmt.Errorf("insertCreatable: not implemented") } -func (m *mockAccountWriter) deleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { +func (m *mockAccountWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { return 0, fmt.Errorf("deleteCreatable: not implemented") } -func (m *mockAccountWriter) close() { +func (m *mockAccountWriter) Close() { } func factorial(n int) int { @@ -3265,15 +2016,15 @@ func TestAccountUnorderedUpdates(t *testing.T) { var baseAccounts lruAccounts baseAccounts.init(nil, 100, 80) - pad, ok, err := mock.lookup(addr1) + pad, ok, err := mock.Lookup(addr1) a.NoError(err) a.True(ok) baseAccounts.write(pad) - pad, ok, err = mock.lookup(observer) + pad, ok, err = mock.Lookup(observer) a.NoError(err) a.True(ok) baseAccounts.write(pad) - baseAccounts.write(persistedAccountData{addr: addr2}) + baseAccounts.write(store.PersistedAccountData{Addr: addr2}) acctDeltas := makeCompactAccountDeltas(updates, dbRound, false, baseAccounts) a.Empty(acctDeltas.misses) @@ -3283,11 +2034,11 @@ func TestAccountUnorderedUpdates(t *testing.T) { var baseResources lruResources baseResources.init(nil, 100, 80) - prd, ok, err := mock.lookupResource(addr1, basics.CreatableIndex(aidx)) + prd, ok, err := mock.LookupResource(addr1, basics.CreatableIndex(aidx)) a.NoError(err) a.True(ok) baseResources.write(prd, addr1) - prd, ok, err = mock.lookupResource(observer, basics.CreatableIndex(aidx)) + prd, ok, err = mock.LookupResource(observer, basics.CreatableIndex(aidx)) a.NoError(err) a.True(ok) baseResources.write(prd, observer) @@ -3358,26 +2109,26 @@ func TestAccountsNewRoundDeletedResourceEntries(t *testing.T) { var baseResources lruResources baseResources.init(nil, 100, 80) - pad, ok, err := mock.lookup(addr1) + pad, ok, err := mock.Lookup(addr1) a.NoError(err) a.True(ok) baseAccounts.write(pad) - pad, ok, err = mock.lookup(observer) + pad, ok, err = mock.Lookup(observer) a.NoError(err) a.True(ok) baseAccounts.write(pad) - baseAccounts.write(persistedAccountData{addr: addr2}) // put an empty record for addr2 to get rid of lookups + baseAccounts.write(store.PersistedAccountData{Addr: addr2}) // put an empty record for addr2 to get rid of lookups acctDeltas := makeCompactAccountDeltas(updates, dbRound, false, baseAccounts) a.Empty(acctDeltas.misses) a.Equal(3, acctDeltas.len()) // we want to have (addr1, aidx) and (observer, aidx) - prd, ok, err := mock.lookupResource(addr1, basics.CreatableIndex(aidx)) + prd, ok, err := mock.LookupResource(addr1, basics.CreatableIndex(aidx)) a.NoError(err) a.True(ok) baseResources.write(prd, addr1) - prd, ok, err = mock.lookupResource(observer, basics.CreatableIndex(aidx)) + prd, ok, err = mock.LookupResource(observer, basics.CreatableIndex(aidx)) a.NoError(err) a.True(ok) baseResources.write(prd, observer) @@ -3399,9 +2150,9 @@ func TestAccountsNewRoundDeletedResourceEntries(t *testing.T) { addressesToCheck := map[basics.Address]bool{addr1: true, addr2: true} matches := 0 for _, upd := range updatedAccounts { - if addressesToCheck[upd.addr] { - a.Equal(int64(0), upd.rowid) - a.Empty(upd.accountData) + if addressesToCheck[upd.Addr] { + a.Equal(int64(0), upd.Rowid) + a.Empty(upd.AccountData) matches++ } } @@ -3410,9 +2161,9 @@ func TestAccountsNewRoundDeletedResourceEntries(t *testing.T) { for addr := range addressesToCheck { upd := updatedResources[addr] a.Equal(1, len(upd)) - a.Equal(int64(0), upd[0].addrid) - a.Equal(basics.CreatableIndex(aidx), upd[0].aidx) - a.Equal(makeResourcesData(uint64(0)), upd[0].data) + a.Equal(int64(0), upd[0].Addrid) + a.Equal(basics.CreatableIndex(aidx), upd[0].Aidx) + a.Equal(store.MakeResourcesData(uint64(0)), upd[0].Data) } } @@ -3420,12 +2171,12 @@ func BenchmarkLRUResources(b *testing.B) { var baseResources lruResources baseResources.init(nil, 1000, 850) - var data persistedResourcesData + var data store.PersistedResourcesData var has bool addrs := make([]basics.Address, 850) for i := 0; i < 850; i++ { - data.data.ApprovalProgram = make([]byte, 8096*4) - data.aidx = basics.CreatableIndex(1) + data.Data.ApprovalProgram = make([]byte, 8096*4) + data.Aidx = basics.CreatableIndex(1) addrBytes := ([]byte(fmt.Sprintf("%d", i)))[:32] var addr basics.Address for i, b := range addrBytes { @@ -3469,17 +2220,17 @@ func initBoxDatabase(b *testing.B, totalBoxes, boxSize int) (db.Pair, func(), er for batch := 0; batch <= batchCount; batch++ { tx, err = dbs.Wdb.Handle.Begin() require.NoError(b, err) - writer, err := makeAccountsSQLWriter(tx, false, false, true, false) + writer, err := store.MakeAccountsSQLWriter(tx, false, false, true, false) require.NoError(b, err) for boxIdx := 0; boxIdx < totalBoxes/batchCount; boxIdx++ { - err = writer.upsertKvPair(fmt.Sprintf("%d", cnt), make([]byte, boxSize)) + err = writer.UpsertKvPair(fmt.Sprintf("%d", cnt), make([]byte, boxSize)) require.NoError(b, err) cnt++ } err = tx.Commit() require.NoError(b, err) - writer.close() + writer.Close() } err = dbs.Wdb.SetSynchronousMode(context.Background(), db.SynchronousModeFull, true) return dbs, cleanup, err @@ -3511,10 +2262,10 @@ func BenchmarkBoxDatabaseRead(b *testing.B) { require.NoError(b, err) var v sql.NullString for i := 0; i < b.N; i++ { - var pv persistedKVData + var pv store.PersistedKVData boxName := boxNames[i%totalBoxes] b.StartTimer() - err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.round, &v) + err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.Round, &v) b.StopTimer() require.NoError(b, err) require.True(b, v.Valid) @@ -3542,9 +2293,9 @@ func BenchmarkBoxDatabaseRead(b *testing.B) { require.NoError(b, err) var v sql.NullString for i := 0; i < b.N+lookback; i++ { - var pv persistedKVData + var pv store.PersistedKVData boxName := boxNames[i%totalBoxes] - err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.round, &v) + err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.Round, &v) require.NoError(b, err) require.True(b, v.Valid) @@ -3552,7 +2303,7 @@ func BenchmarkBoxDatabaseRead(b *testing.B) { if i >= lookback { boxName = boxNames[(i-lookback)%totalBoxes] b.StartTimer() - err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.round, &v) + err = lookupStmt.QueryRow([]byte(fmt.Sprintf("%d", boxName))).Scan(&pv.Round, &v) b.StopTimer() require.NoError(b, err) require.True(b, v.Valid) @@ -3701,7 +2452,7 @@ func TestAccountOnlineQueries(t *testing.T) { addRound(2, delta2) addRound(3, delta3) - queries, err := onlineAccountsInitDbQueries(tx) + queries, err := store.OnlineAccountsInitDbQueries(tx) require.NoError(t, err) // check round 1 @@ -3723,25 +2474,25 @@ func TestAccountOnlineQueries(t *testing.T) { require.Equal(t, addrB, onlineAcctB.Address) require.Equal(t, dataB1.AccountBaseData.MicroAlgos, onlineAcctB.MicroAlgos) - paod, err := queries.lookupOnline(addrA, rnd) + paod, err := queries.LookupOnline(addrA, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrA, paod.addr) - require.Equal(t, dataA1.AccountBaseData.MicroAlgos, paod.accountData.MicroAlgos) - require.Equal(t, voteIDA, paod.accountData.VoteID) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrA, paod.Addr) + require.Equal(t, dataA1.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos) + require.Equal(t, voteIDA, paod.AccountData.VoteID) - paod, err = queries.lookupOnline(addrB, rnd) + paod, err = queries.LookupOnline(addrB, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrB, paod.addr) - require.Equal(t, dataB1.AccountBaseData.MicroAlgos, paod.accountData.MicroAlgos) - require.Equal(t, voteIDB, paod.accountData.VoteID) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrB, paod.Addr) + require.Equal(t, dataB1.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos) + require.Equal(t, voteIDB, paod.AccountData.VoteID) - paod, err = queries.lookupOnline(addrC, rnd) + paod, err = queries.LookupOnline(addrC, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrC, paod.addr) - require.Empty(t, paod.accountData) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrC, paod.Addr) + require.Empty(t, paod.AccountData) // check round 2 rnd = basics.Round(2) @@ -3757,24 +2508,24 @@ func TestAccountOnlineQueries(t *testing.T) { require.Equal(t, addrB, onlineAcctB.Address) require.Equal(t, dataB1.AccountBaseData.MicroAlgos, onlineAcctB.MicroAlgos) - paod, err = queries.lookupOnline(addrA, rnd) + paod, err = queries.LookupOnline(addrA, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrA, paod.addr) - require.Empty(t, paod.accountData) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrA, paod.Addr) + require.Empty(t, paod.AccountData) - paod, err = queries.lookupOnline(addrB, rnd) + paod, err = queries.LookupOnline(addrB, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrB, paod.addr) - require.Equal(t, dataB1.AccountBaseData.MicroAlgos, paod.accountData.MicroAlgos) - require.Equal(t, voteIDB, paod.accountData.VoteID) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrB, paod.Addr) + require.Equal(t, dataB1.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos) + require.Equal(t, voteIDB, paod.AccountData.VoteID) - paod, err = queries.lookupOnline(addrC, rnd) + paod, err = queries.LookupOnline(addrC, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrC, paod.addr) - require.Empty(t, paod.accountData) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrC, paod.Addr) + require.Empty(t, paod.AccountData) // check round 3 rnd = basics.Round(3) @@ -3790,24 +2541,24 @@ func TestAccountOnlineQueries(t *testing.T) { require.Equal(t, addrC, onlineAcctC.Address) require.Equal(t, dataC3.AccountBaseData.MicroAlgos, onlineAcctC.MicroAlgos) - paod, err = queries.lookupOnline(addrA, rnd) + paod, err = queries.LookupOnline(addrA, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrA, paod.addr) - require.Empty(t, paod.accountData) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrA, paod.Addr) + require.Empty(t, paod.AccountData) - paod, err = queries.lookupOnline(addrB, rnd) + paod, err = queries.LookupOnline(addrB, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrB, paod.addr) - require.Empty(t, paod.accountData) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrB, paod.Addr) + require.Empty(t, paod.AccountData) - paod, err = queries.lookupOnline(addrC, rnd) + paod, err = queries.LookupOnline(addrC, rnd) require.NoError(t, err) - require.Equal(t, basics.Round(3), paod.round) - require.Equal(t, addrC, paod.addr) - require.Equal(t, dataC3.AccountBaseData.MicroAlgos, paod.accountData.MicroAlgos) - require.Equal(t, voteIDC, paod.accountData.VoteID) + require.Equal(t, basics.Round(3), paod.Round) + require.Equal(t, addrC, paod.Addr) + require.Equal(t, dataC3.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos) + require.Equal(t, voteIDC, paod.AccountData.VoteID) paods, err := onlineAccountsAll(tx, 0) require.NoError(t, err) @@ -3824,27 +2575,27 @@ func TestAccountOnlineQueries(t *testing.T) { // A | 2 | 0 checkAddrB := func() { - require.Equal(t, int64(2), paods[0].rowid) - require.Equal(t, basics.Round(1), paods[0].updRound) - require.Equal(t, addrB, paods[0].addr) - require.Equal(t, int64(4), paods[1].rowid) - require.Equal(t, basics.Round(3), paods[1].updRound) - require.Equal(t, addrB, paods[1].addr) + require.Equal(t, int64(2), paods[0].Rowid) + require.Equal(t, basics.Round(1), paods[0].UpdRound) + require.Equal(t, addrB, paods[0].Addr) + require.Equal(t, int64(4), paods[1].Rowid) + require.Equal(t, basics.Round(3), paods[1].UpdRound) + require.Equal(t, addrB, paods[1].Addr) } checkAddrC := func() { - require.Equal(t, int64(5), paods[2].rowid) - require.Equal(t, basics.Round(3), paods[2].updRound) - require.Equal(t, addrC, paods[2].addr) + require.Equal(t, int64(5), paods[2].Rowid) + require.Equal(t, basics.Round(3), paods[2].UpdRound) + require.Equal(t, addrC, paods[2].Addr) } checkAddrA := func() { - require.Equal(t, int64(1), paods[3].rowid) - require.Equal(t, basics.Round(1), paods[3].updRound) - require.Equal(t, addrA, paods[3].addr) - require.Equal(t, int64(3), paods[4].rowid) - require.Equal(t, basics.Round(2), paods[4].updRound) - require.Equal(t, addrA, paods[4].addr) + require.Equal(t, int64(1), paods[3].Rowid) + require.Equal(t, basics.Round(1), paods[3].UpdRound) + require.Equal(t, addrA, paods[3].Addr) + require.Equal(t, int64(3), paods[4].Rowid) + require.Equal(t, basics.Round(2), paods[4].UpdRound) + require.Equal(t, addrA, paods[4].Addr) } checkAddrB() @@ -3869,42 +2620,42 @@ func TestAccountOnlineQueries(t *testing.T) { require.Equal(t, 2, len(paods)) checkAddrB() - paods, rnd, err = queries.lookupOnlineHistory(addrA) + paods, rnd, err = queries.LookupOnlineHistory(addrA) require.NoError(t, err) require.Equal(t, basics.Round(3), rnd) require.Equal(t, 2, len(paods)) - require.Equal(t, int64(1), paods[0].rowid) - require.Equal(t, basics.Round(1), paods[0].updRound) - require.Equal(t, int64(3), paods[1].rowid) - require.Equal(t, basics.Round(2), paods[1].updRound) + require.Equal(t, int64(1), paods[0].Rowid) + require.Equal(t, basics.Round(1), paods[0].UpdRound) + require.Equal(t, int64(3), paods[1].Rowid) + require.Equal(t, basics.Round(2), paods[1].UpdRound) - paods, rnd, err = queries.lookupOnlineHistory(addrB) + paods, rnd, err = queries.LookupOnlineHistory(addrB) require.NoError(t, err) require.Equal(t, basics.Round(3), rnd) require.Equal(t, 2, len(paods)) - require.Equal(t, int64(2), paods[0].rowid) - require.Equal(t, basics.Round(1), paods[0].updRound) - require.Equal(t, int64(4), paods[1].rowid) - require.Equal(t, basics.Round(3), paods[1].updRound) + require.Equal(t, int64(2), paods[0].Rowid) + require.Equal(t, basics.Round(1), paods[0].UpdRound) + require.Equal(t, int64(4), paods[1].Rowid) + require.Equal(t, basics.Round(3), paods[1].UpdRound) - paods, rnd, err = queries.lookupOnlineHistory(addrC) + paods, rnd, err = queries.LookupOnlineHistory(addrC) require.NoError(t, err) require.Equal(t, basics.Round(3), rnd) require.Equal(t, 1, len(paods)) - require.Equal(t, int64(5), paods[0].rowid) - require.Equal(t, basics.Round(3), paods[0].updRound) + require.Equal(t, int64(5), paods[0].Rowid) + require.Equal(t, basics.Round(3), paods[0].UpdRound) } type mockOnlineAccountsWriter struct { rowid int64 } -func (w *mockOnlineAccountsWriter) insertOnlineAccount(addr basics.Address, normBalance uint64, data baseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) { +func (w *mockOnlineAccountsWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data store.BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) { w.rowid++ return w.rowid, nil } -func (w *mockOnlineAccountsWriter) close() {} +func (w *mockOnlineAccountsWriter) Close() {} func TestAccountOnlineAccountsNewRound(t *testing.T) { partitiontest.PartitionTest(t) @@ -3926,7 +2677,7 @@ func TestAccountOnlineAccountsNewRound(t *testing.T) { // acct B is new and offline deltaB := onlineAccountDelta{ address: addrB, - newAcct: []baseOnlineAccountData{{ + newAcct: []store.BaseOnlineAccountData{{ MicroAlgos: basics.MicroAlgos{Raw: 200_000_000}, }}, updRound: []uint64{1}, @@ -3935,9 +2686,9 @@ func TestAccountOnlineAccountsNewRound(t *testing.T) { // acct C is new and online deltaC := onlineAccountDelta{ address: addrC, - newAcct: []baseOnlineAccountData{{ + newAcct: []store.BaseOnlineAccountData{{ MicroAlgos: basics.MicroAlgos{Raw: 300_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 500}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 500}, }}, newStatus: []basics.Status{basics.Online}, updRound: []uint64{2}, @@ -3945,15 +2696,15 @@ func TestAccountOnlineAccountsNewRound(t *testing.T) { // acct D is old and went offline deltaD := onlineAccountDelta{ address: addrD, - oldAcct: persistedOnlineAccountData{ - addr: addrD, - accountData: baseOnlineAccountData{ + oldAcct: store.PersistedOnlineAccountData{ + Addr: addrD, + AccountData: store.BaseOnlineAccountData{ MicroAlgos: basics.MicroAlgos{Raw: 400_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 500}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 500}, }, - rowid: 1, + Rowid: 1, }, - newAcct: []baseOnlineAccountData{{ + newAcct: []store.BaseOnlineAccountData{{ MicroAlgos: basics.MicroAlgos{Raw: 400_000_000}, }}, newStatus: []basics.Status{basics.Offline}, @@ -3963,17 +2714,17 @@ func TestAccountOnlineAccountsNewRound(t *testing.T) { // acct E is old online deltaE := onlineAccountDelta{ address: addrE, - oldAcct: persistedOnlineAccountData{ - addr: addrE, - accountData: baseOnlineAccountData{ + oldAcct: store.PersistedOnlineAccountData{ + Addr: addrE, + AccountData: store.BaseOnlineAccountData{ MicroAlgos: basics.MicroAlgos{Raw: 500_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 500}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 500}, }, - rowid: 2, + Rowid: 2, }, - newAcct: []baseOnlineAccountData{{ + newAcct: []store.BaseOnlineAccountData{{ MicroAlgos: basics.MicroAlgos{Raw: 500_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 600}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 600}, }}, newStatus: []basics.Status{basics.Online}, updRound: []uint64{4}, @@ -3985,9 +2736,9 @@ func TestAccountOnlineAccountsNewRound(t *testing.T) { require.NoError(t, err) require.Len(t, updated, 3) - require.Equal(t, updated[0].addr, addrC) - require.Equal(t, updated[1].addr, addrD) - require.Equal(t, updated[2].addr, addrE) + require.Equal(t, updated[0].Addr, addrC) + require.Equal(t, updated[1].Addr, addrD) + require.Equal(t, updated[2].Addr, addrE) // check errors: new online with empty voting data deltaC.newStatus[0] = basics.Online @@ -4024,13 +2775,13 @@ func TestAccountOnlineAccountsNewRoundFlip(t *testing.T) { // acct A is new, offline and then online deltaA := onlineAccountDelta{ address: addrA, - newAcct: []baseOnlineAccountData{ + newAcct: []store.BaseOnlineAccountData{ { MicroAlgos: basics.MicroAlgos{Raw: 100_000_000}, }, { MicroAlgos: basics.MicroAlgos{Raw: 100_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 100}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 100}, }, }, updRound: []uint64{1, 2}, @@ -4039,10 +2790,10 @@ func TestAccountOnlineAccountsNewRoundFlip(t *testing.T) { // acct B is new and online and then offline deltaB := onlineAccountDelta{ address: addrB, - newAcct: []baseOnlineAccountData{ + newAcct: []store.BaseOnlineAccountData{ { MicroAlgos: basics.MicroAlgos{Raw: 200_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 200}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 200}, }, { MicroAlgos: basics.MicroAlgos{Raw: 200_000_000}, @@ -4054,18 +2805,18 @@ func TestAccountOnlineAccountsNewRoundFlip(t *testing.T) { // acct C is old online, then online and then offline deltaC := onlineAccountDelta{ address: addrC, - oldAcct: persistedOnlineAccountData{ - addr: addrC, - accountData: baseOnlineAccountData{ + oldAcct: store.PersistedOnlineAccountData{ + Addr: addrC, + AccountData: store.BaseOnlineAccountData{ MicroAlgos: basics.MicroAlgos{Raw: 300_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 300}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 300}, }, - rowid: 1, + Rowid: 1, }, - newAcct: []baseOnlineAccountData{ + newAcct: []store.BaseOnlineAccountData{ { MicroAlgos: basics.MicroAlgos{Raw: 300_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 301}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 301}, }, { MicroAlgos: basics.MicroAlgos{Raw: 300_000_000}, @@ -4081,11 +2832,11 @@ func TestAccountOnlineAccountsNewRoundFlip(t *testing.T) { require.NoError(t, err) require.Len(t, updated, 5) - require.Equal(t, updated[0].addr, addrA) - require.Equal(t, updated[1].addr, addrB) - require.Equal(t, updated[2].addr, addrB) - require.Equal(t, updated[3].addr, addrC) - require.Equal(t, updated[4].addr, addrC) + require.Equal(t, updated[0].Addr, addrA) + require.Equal(t, updated[1].Addr, addrB) + require.Equal(t, updated[2].Addr, addrB) + require.Equal(t, updated[3].Addr, addrC) + require.Equal(t, updated[4].Addr, addrC) } func TestAccountOnlineRoundParams(t *testing.T) { @@ -4262,17 +3013,17 @@ func TestOnlineAccountsDeletion(t *testing.T) { deltaA := onlineAccountDelta{ address: addrA, - newAcct: []baseOnlineAccountData{ + newAcct: []store.BaseOnlineAccountData{ { MicroAlgos: basics.MicroAlgos{Raw: 100_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 100}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 100}, }, { MicroAlgos: basics.MicroAlgos{Raw: 100_000_000}, }, { MicroAlgos: basics.MicroAlgos{Raw: 100_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 600}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 600}, }, }, updRound: []uint64{1, 3, 6}, @@ -4281,14 +3032,14 @@ func TestOnlineAccountsDeletion(t *testing.T) { // acct B is new and online and then offline deltaB := onlineAccountDelta{ address: addrB, - newAcct: []baseOnlineAccountData{ + newAcct: []store.BaseOnlineAccountData{ { MicroAlgos: basics.MicroAlgos{Raw: 200_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 300}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 300}, }, { MicroAlgos: basics.MicroAlgos{Raw: 200_000_000}, - baseVotingData: baseVotingData{VoteFirstValid: 700}, + BaseVotingData: store.BaseVotingData{VoteFirstValid: 700}, }, }, updRound: []uint64{3, 7}, @@ -4296,11 +3047,11 @@ func TestOnlineAccountsDeletion(t *testing.T) { } updates.deltas = append(updates.deltas, deltaA, deltaB) - writer, err := makeOnlineAccountsSQLWriter(tx, updates.len() > 0) + writer, err := store.MakeOnlineAccountsSQLWriter(tx, updates.len() > 0) if err != nil { return } - defer writer.close() + defer writer.Close() lastUpdateRound := basics.Round(10) proto := config.Consensus[protocol.ConsensusCurrentVersion] @@ -4308,11 +3059,11 @@ func TestOnlineAccountsDeletion(t *testing.T) { require.NoError(t, err) require.Len(t, updated, 5) - queries, err := onlineAccountsInitDbQueries(tx) + queries, err := store.OnlineAccountsInitDbQueries(tx) require.NoError(t, err) var count int64 - var history []persistedOnlineAccountData + var history []store.PersistedOnlineAccountData var validThrough basics.Round for _, rnd := range []basics.Round{1, 2, 3} { err = onlineAccountsDelete(tx, rnd) @@ -4322,11 +3073,11 @@ func TestOnlineAccountsDeletion(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(5), count) - history, validThrough, err = queries.lookupOnlineHistory(addrA) + history, validThrough, err = queries.LookupOnlineHistory(addrA) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) // not set require.Len(t, history, 3) - history, validThrough, err = queries.lookupOnlineHistory(addrB) + history, validThrough, err = queries.LookupOnlineHistory(addrB) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) require.Len(t, history, 2) @@ -4340,11 +3091,11 @@ func TestOnlineAccountsDeletion(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(3), count) - history, validThrough, err = queries.lookupOnlineHistory(addrA) + history, validThrough, err = queries.LookupOnlineHistory(addrA) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) require.Len(t, history, 1) - history, validThrough, err = queries.lookupOnlineHistory(addrB) + history, validThrough, err = queries.LookupOnlineHistory(addrB) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) require.Len(t, history, 2) @@ -4358,11 +3109,11 @@ func TestOnlineAccountsDeletion(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(2), count) - history, validThrough, err = queries.lookupOnlineHistory(addrA) + history, validThrough, err = queries.LookupOnlineHistory(addrA) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) require.Len(t, history, 1) - history, validThrough, err = queries.lookupOnlineHistory(addrB) + history, validThrough, err = queries.LookupOnlineHistory(addrB) require.NoError(t, err) require.Equal(t, basics.Round(0), validThrough) require.Len(t, history, 1) @@ -4532,7 +3283,7 @@ func TestRemoveOfflineStateProofID(t *testing.T) { err = rows.Scan(&addrbuf, &encodedAcctData) require.NoError(t, err) copy(addr[:], addrbuf) - var ba baseAccountData + var ba store.BaseAccountData err = protocol.Decode(encodedAcctData, &ba) require.NoError(t, err) if expected && ba.Status != basics.Online { @@ -4582,7 +3333,7 @@ func TestRemoveOfflineStateProofID(t *testing.T) { var encodedAcctData []byte err = rows.Scan(&addrid, &encodedAcctData) require.NoError(t, err) - var ba baseAccountData + var ba store.BaseAccountData err = protocol.Decode(encodedAcctData, &ba) require.NoError(t, err) if ba.Status != basics.Online { @@ -4591,8 +3342,8 @@ func TestRemoveOfflineStateProofID(t *testing.T) { } } -func randomBaseAccountData() baseAccountData { - vd := baseVotingData{ +func randomBaseAccountData() store.BaseAccountData { + vd := store.BaseVotingData{ VoteFirstValid: basics.Round(crypto.RandUint64()), VoteLastValid: basics.Round(crypto.RandUint64()), VoteKeyDilution: crypto.RandUint64(), @@ -4601,7 +3352,7 @@ func randomBaseAccountData() baseAccountData { crypto.RandBytes(vd.StateProofID[:]) crypto.RandBytes(vd.SelectionID[:]) - baseAD := baseAccountData{ + baseAD := store.BaseAccountData{ Status: basics.Online, MicroAlgos: basics.MicroAlgos{Raw: crypto.RandUint64()}, RewardsBase: crypto.RandUint64(), @@ -4614,7 +3365,7 @@ func randomBaseAccountData() baseAccountData { TotalAssets: crypto.RandUint64(), TotalAppParams: crypto.RandUint64(), TotalAppLocalStates: crypto.RandUint64(), - baseVotingData: vd, + BaseVotingData: vd, UpdateRound: crypto.RandUint64(), } @@ -4638,12 +3389,12 @@ func makeString(len int) string { return s } -func randomAssetResourceData() resourcesData { +func randomAssetResourceData() store.ResourcesData { currentConsensusParams := config.Consensus[protocol.ConsensusCurrentVersion] // resourcesData is suiteable for keeping asset params, holding, app params, app local state // but only asset + holding or app + local state can appear there - rdAsset := resourcesData{ + rdAsset := store.ResourcesData{ Total: crypto.RandUint64(), Decimals: uint32(crypto.RandUint63() % uint64(math.MaxUint32)), DefaultFrozen: true, @@ -4664,10 +3415,10 @@ func randomAssetResourceData() resourcesData { return rdAsset } -func randomAppResourceData() resourcesData { +func randomAppResourceData() store.ResourcesData { currentConsensusParams := config.Consensus[protocol.ConsensusCurrentVersion] - rdApp := resourcesData{ + rdApp := store.ResourcesData{ SchemaNumUint: crypto.RandUint64(), SchemaNumByteSlice: crypto.RandUint64(), diff --git a/ledger/acctonline.go b/ledger/acctonline.go index e9a20046f3..c0cea58516 100644 --- a/ledger/acctonline.go +++ b/ledger/acctonline.go @@ -33,6 +33,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/metrics" @@ -51,9 +52,10 @@ type modifiedOnlineAccount struct { } // cachedOnlineAccount is a light-weight version of persistedOnlineAccountData suitable for in-memory caching +// //msgp:ignore cachedOnlineAccount type cachedOnlineAccount struct { - baseOnlineAccountData + store.BaseOnlineAccountData updRound basics.Round } @@ -63,7 +65,7 @@ type onlineAccounts struct { dbs db.Pair // Prepared SQL statements for fast accounts DB lookups. - accountsq *onlineAccountsDbQueries + accountsq store.OnlineAccountsReader // cachedDBRoundOnline is always exactly tracker DB round (and therefore, onlineAccountsRound()), // cached to use in lookup functions @@ -172,7 +174,7 @@ func (ao *onlineAccounts) initializeFromDisk(l ledgerForTracker, lastBalancesRou return } - ao.accountsq, err = onlineAccountsInitDbQueries(ao.dbs.Rdb.Handle) + ao.accountsq, err = store.OnlineAccountsInitDbQueries(ao.dbs.Rdb.Handle) if err != nil { return } @@ -193,7 +195,7 @@ func (ao *onlineAccounts) latest() basics.Round { // close closes the accountUpdates, waiting for all the child go-routine to complete func (ao *onlineAccounts) close() { if ao.accountsq != nil { - ao.accountsq.close() + ao.accountsq.Close() ao.accountsq = nil } @@ -463,10 +465,10 @@ func (ao *onlineAccounts) postCommit(ctx context.Context, dcc *deferredCommitCon for _, persistedAcct := range dcc.updatedPersistedOnlineAccounts { ao.baseOnlineAccounts.write(persistedAcct) ao.onlineAccountsCache.writeFrontIfExist( - persistedAcct.addr, + persistedAcct.Addr, cachedOnlineAccount{ - baseOnlineAccountData: persistedAcct.accountData, - updRound: persistedAcct.updRound, + BaseOnlineAccountData: persistedAcct.AccountData, + updRound: persistedAcct.UpdRound, }) } @@ -526,7 +528,7 @@ func (ao *onlineAccounts) onlineTotalsEx(rnd basics.Round) (basics.MicroAlgos, e ao.log.Errorf("onlineTotalsImpl error: %v", err) } - totalsOnline, err = ao.accountsq.lookupOnlineTotalsHistory(rnd) + totalsOnline, err = ao.accountsq.LookupOnlineTotalsHistory(rnd) return totalsOnline, err } @@ -615,7 +617,7 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics. var paramsOffset uint64 var rewardsProto config.ConsensusParams var rewardsLevel uint64 - var persistedData persistedOnlineAccountData + var persistedData store.PersistedOnlineAccountData // the loop serves retrying logic if the database advanced while // the function was analyzing deltas or caches. @@ -678,8 +680,8 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics. // As an optimization, we avoid creating // a separate transaction here, and directly use a prepared SQL query // against the database. - persistedData, err = ao.accountsq.lookupOnline(addr, rnd) - if err != nil || persistedData.rowid == 0 { + persistedData, err = ao.accountsq.LookupOnline(addr, rnd) + if err != nil || persistedData.Rowid == 0 { // no such online account, return empty return ledgercore.OnlineAccountData{}, err } @@ -694,7 +696,7 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics. // * if commitRound deletes some history after, the cache has additional entries and updRound comparison gets a right value // 2. after commitRound but before postCommit => OK, read full history, ignore the update from postCommit in writeFront's updRound comparison // 3. after postCommit => OK, postCommit does not add new entry with writeFrontIfExist, but here all the full history is loaded - persistedDataHistory, validThrough, err := ao.accountsq.lookupOnlineHistory(addr) + persistedDataHistory, validThrough, err := ao.accountsq.LookupOnlineHistory(addr) if err != nil || len(persistedDataHistory) == 0 { return ledgercore.OnlineAccountData{}, err } @@ -715,21 +717,21 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics. } else { for _, data := range persistedDataHistory { written := ao.onlineAccountsCache.writeFront( - data.addr, + data.Addr, cachedOnlineAccount{ - baseOnlineAccountData: data.accountData, - updRound: data.updRound, + BaseOnlineAccountData: data.AccountData, + updRound: data.UpdRound, }) if !written { ao.accountsMu.Unlock() - err = fmt.Errorf("failed to write history of acct %s for round %d into online accounts cache", data.addr.String(), data.updRound) + err = fmt.Errorf("failed to write history of acct %s for round %d into online accounts cache", data.Addr.String(), data.UpdRound) return ledgercore.OnlineAccountData{}, err } } ao.log.Info("inserted new item to onlineAccountsCache") } ao.accountsMu.Unlock() - return persistedData.accountData.GetOnlineAccountData(rewardsProto, rewardsLevel), nil + return persistedData.AccountData.GetOnlineAccountData(rewardsProto, rewardsLevel), nil } // case 3.3: retry (for loop iterates and queries again) ao.accountsMu.Unlock() diff --git a/ledger/acctonline_test.go b/ledger/acctonline_test.go index 67871c23e9..c1e0c2fbde 100644 --- a/ledger/acctonline_test.go +++ b/ledger/acctonline_test.go @@ -180,14 +180,14 @@ func TestAcctOnline(t *testing.T) { require.NoError(t, err) for _, bal := range allAccts { - data, err := oa.accountsq.lookupOnline(bal.Addr, 0) + data, err := oa.accountsq.LookupOnline(bal.Addr, 0) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.Equal(t, basics.Round(0), data.round) - require.Equal(t, bal.AccountData.MicroAlgos, data.accountData.MicroAlgos) - require.Equal(t, bal.AccountData.RewardsBase, data.accountData.RewardsBase) - require.Equal(t, bal.AccountData.VoteFirstValid, data.accountData.VoteFirstValid) - require.Equal(t, bal.AccountData.VoteLastValid, data.accountData.VoteLastValid) + require.Equal(t, bal.Addr, data.Addr) + require.Equal(t, basics.Round(0), data.Round) + require.Equal(t, bal.AccountData.MicroAlgos, data.AccountData.MicroAlgos) + require.Equal(t, bal.AccountData.RewardsBase, data.AccountData.RewardsBase) + require.Equal(t, bal.AccountData.VoteFirstValid, data.AccountData.VoteFirstValid) + require.Equal(t, bal.AccountData.VoteLastValid, data.AccountData.VoteLastValid) oad, err := oa.lookupOnlineAccountData(0, bal.Addr) require.NoError(t, err) @@ -219,29 +219,29 @@ func TestAcctOnline(t *testing.T) { rnd := i - basics.Round(maxDeltaLookback) acctIdx := int(rnd) - 1 bal := allAccts[acctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.Empty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.Empty(t, data.AccountData) data, has := oa.baseOnlineAccounts.read(bal.Addr) require.True(t, has) - require.NotEmpty(t, data.rowid) - require.Empty(t, data.accountData) + require.NotEmpty(t, data.Rowid) + require.Empty(t, data.AccountData) oad, err := oa.lookupOnlineAccountData(rnd, bal.Addr) require.NoError(t, err) require.Empty(t, oad) // check the prev original row is still there - data, err = oa.accountsq.lookupOnline(bal.Addr, rnd-1) + data, err = oa.accountsq.LookupOnline(bal.Addr, rnd-1) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.NotEmpty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.NotEmpty(t, data.AccountData) } // check data gets expired and removed from the DB @@ -252,17 +252,17 @@ func TestAcctOnline(t *testing.T) { rnd := i - basics.Round(maxBalLookback+maxDeltaLookback) acctIdx := int(rnd) - 1 bal := allAccts[acctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.Empty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.Empty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.Empty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.Empty(t, data.AccountData) data, has := oa.baseOnlineAccounts.read(bal.Addr) require.True(t, has) - require.NotEmpty(t, data.rowid) // TODO: FIXME: set rowid to empty for these items - require.Empty(t, data.accountData) + require.NotEmpty(t, data.Rowid) // TODO: FIXME: set rowid to empty for these items + require.Empty(t, data.AccountData) // committed round i => dbRound = i - maxDeltaLookback (= 13 for the account 0) // dbRound - maxBalLookback (= 1) is the "set offline" round for account 0 @@ -277,18 +277,18 @@ func TestAcctOnline(t *testing.T) { nextAcctIdx := acctIdx + 1 if nextAcctIdx < int(targetRound) { bal := allAccts[nextAcctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.NotEmpty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.NotEmpty(t, data.AccountData) // the most recent value is empty because the account is scheduled for removal data, has := oa.baseOnlineAccounts.read(bal.Addr) require.True(t, has) - require.NotEmpty(t, data.rowid) // TODO: FIXME: set rowid to empty for these items - require.Empty(t, data.accountData) + require.NotEmpty(t, data.Rowid) // TODO: FIXME: set rowid to empty for these items + require.Empty(t, data.AccountData) // account 1 went offline at round 2 => it offline at requested round 1+1=2 oad, err := oa.lookupOnlineAccountData(rnd+1, bal.Addr) @@ -301,18 +301,18 @@ func TestAcctOnline(t *testing.T) { nextNextAcctIdx := nextAcctIdx + 1 if nextNextAcctIdx < int(targetRound) { bal := allAccts[nextNextAcctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.NotEmpty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.NotEmpty(t, data.AccountData) // the most recent value is empty because the account is scheduled for removal data, has := oa.baseOnlineAccounts.read(bal.Addr) require.True(t, has) - require.NotEmpty(t, data.rowid) // TODO: FIXME: set rowid to empty for these items - require.Empty(t, data.accountData) + require.NotEmpty(t, data.Rowid) // TODO: FIXME: set rowid to empty for these items + require.Empty(t, data.AccountData) // account 2 went offline at round 3 => it online at requested round 1+1=2 oad, err := oa.lookupOnlineAccountData(rnd+1, bal.Addr) @@ -331,29 +331,29 @@ func TestAcctOnline(t *testing.T) { for i := numPersistedAccounts - maxBalLookback; i < numPersistedAccounts; i++ { bal := allAccts[i] // we expire account i at round i+1 - data, err := oa.accountsq.lookupOnline(bal.Addr, basics.Round(i+1)) + data, err := oa.accountsq.LookupOnline(bal.Addr, basics.Round(i+1)) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.Empty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.Empty(t, data.AccountData) data, has := oa.baseOnlineAccounts.read(bal.Addr) require.True(t, has) - require.NotEmpty(t, data.rowid) - require.Empty(t, data.accountData) + require.NotEmpty(t, data.Rowid) + require.Empty(t, data.AccountData) oad, err := oa.lookupOnlineAccountData(basics.Round(i+1), bal.Addr) require.NoError(t, err) require.Empty(t, oad) // ensure the online entry is still in the DB for the round i - data, err = oa.accountsq.lookupOnline(bal.Addr, basics.Round(i)) + data, err = oa.accountsq.LookupOnline(bal.Addr, basics.Round(i)) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.NotEmpty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.NotEmpty(t, data.AccountData) } // check maxDeltaLookback accounts in in-memory deltas, check it @@ -364,12 +364,12 @@ func TestAcctOnline(t *testing.T) { require.Empty(t, oad) // the table has old values b/c not committed yet - data, err := oa.accountsq.lookupOnline(bal.Addr, basics.Round(i)) + data, err := oa.accountsq.LookupOnline(bal.Addr, basics.Round(i)) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) - require.NotEmpty(t, data.accountData) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) + require.NotEmpty(t, data.AccountData) // the base cache also does not have such entires data, has := oa.baseOnlineAccounts.read(bal.Addr) @@ -507,23 +507,23 @@ func TestAcctOnlineCache(t *testing.T) { rnd := i - basics.Round(maxDeltaLookback) acctIdx := (int(rnd) - 1) % numAccts bal := allAccts[acctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.NotEmpty(t, data.rowid) - require.Equal(t, oa.cachedDBRoundOnline, data.round) + require.Equal(t, bal.Addr, data.Addr) + require.NotEmpty(t, data.Rowid) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) if (rnd-1)%(numAccts*2) >= numAccts { - require.Empty(t, data.accountData) + require.Empty(t, data.AccountData) } else { - require.NotEmpty(t, data.accountData) + require.NotEmpty(t, data.AccountData) } cachedData, has := oa.onlineAccountsCache.read(bal.Addr, rnd) require.True(t, has) if (rnd-1)%(numAccts*2) >= numAccts { - require.Empty(t, cachedData.baseOnlineAccountData) + require.Empty(t, cachedData.BaseOnlineAccountData) } else { - require.NotEmpty(t, cachedData.baseOnlineAccountData) + require.NotEmpty(t, cachedData.BaseOnlineAccountData) } oad, err := oa.lookupOnlineAccountData(rnd, bal.Addr) @@ -540,24 +540,24 @@ func TestAcctOnlineCache(t *testing.T) { rnd := i - basics.Round(maxBalLookback+maxDeltaLookback) acctIdx := (int(rnd) - 1) % numAccts bal := allAccts[acctIdx] - data, err := oa.accountsq.lookupOnline(bal.Addr, rnd) + data, err := oa.accountsq.LookupOnline(bal.Addr, rnd) require.NoError(t, err) - require.Equal(t, bal.Addr, data.addr) - require.Equal(t, oa.cachedDBRoundOnline, data.round) + require.Equal(t, bal.Addr, data.Addr) + require.Equal(t, oa.cachedDBRoundOnline, data.Round) if (rnd-1)%(numAccts*2) >= numAccts { - require.Empty(t, data.accountData) - require.Empty(t, data.rowid) + require.Empty(t, data.AccountData) + require.Empty(t, data.Rowid) } else { - require.NotEmpty(t, data.rowid) - require.NotEmpty(t, data.accountData) + require.NotEmpty(t, data.Rowid) + require.NotEmpty(t, data.AccountData) } cachedData, has := oa.onlineAccountsCache.read(bal.Addr, rnd) require.True(t, has) if (rnd-1)%(numAccts*2) >= numAccts { - require.Empty(t, cachedData.baseOnlineAccountData) + require.Empty(t, cachedData.BaseOnlineAccountData) } else { - require.NotEmpty(t, cachedData.baseOnlineAccountData) + require.NotEmpty(t, cachedData.BaseOnlineAccountData) } // committed round i => dbRound = i - maxDeltaLookback @@ -573,7 +573,7 @@ func TestAcctOnlineCache(t *testing.T) { } require.Equal(t, targetRound-basics.Round(maxDeltaLookback), oa.cachedDBRoundOnline) - res, validThrough, err := oa.accountsq.lookupOnlineHistory(addrA) + res, validThrough, err := oa.accountsq.LookupOnlineHistory(addrA) require.NoError(t, err) require.Equal(t, oa.cachedDBRoundOnline, validThrough) // +1 because of deletion before X, and not checking acct state at X @@ -581,10 +581,10 @@ func TestAcctOnlineCache(t *testing.T) { // ensure the cache length corresponds to DB require.Equal(t, len(res), oa.onlineAccountsCache.accounts[addrA].Len()) for _, entry := range res { - cached, has := oa.onlineAccountsCache.read(addrA, entry.updRound) + cached, has := oa.onlineAccountsCache.read(addrA, entry.UpdRound) require.True(t, has) - require.Equal(t, entry.updRound, cached.updRound) - require.Equal(t, entry.accountData.VoteLastValid, cached.VoteLastValid) + require.Equal(t, entry.UpdRound, cached.updRound) + require.Equal(t, entry.AccountData.VoteLastValid, cached.VoteLastValid) } // ensure correct behavior after deleting cache @@ -616,7 +616,7 @@ func TestAcctOnlineCache(t *testing.T) { cachedData, has := oa.onlineAccountsCache.read(bal.Addr, oldRound) require.True(t, has) require.Equal(t, expectedRound, cachedData.updRound) - require.NotEmpty(t, cachedData.baseOnlineAccountData) + require.NotEmpty(t, cachedData.BaseOnlineAccountData) // cache should contain data for new rounds // (the last entry should be offline) @@ -625,7 +625,7 @@ func TestAcctOnlineCache(t *testing.T) { cachedData, has = oa.onlineAccountsCache.read(bal.Addr, newRound) require.True(t, has) require.Equal(t, newRound, cachedData.updRound) - require.Empty(t, cachedData.baseOnlineAccountData) + require.Empty(t, cachedData.BaseOnlineAccountData) }) } @@ -1038,11 +1038,11 @@ func TestAcctOnlineCacheDBSync(t *testing.T) { require.NoError(t, err) require.Empty(t, data.VotingData.VoteLastValid) // ensure offline entry is in DB as well - pad, err := oa.accountsq.lookupOnline(addrA, 1) + pad, err := oa.accountsq.LookupOnline(addrA, 1) require.NoError(t, err) - require.Equal(t, addrA, pad.addr) - require.NotEmpty(t, pad.rowid) - require.Empty(t, pad.accountData.VoteLastValid) + require.Equal(t, addrA, pad.Addr) + require.NotEmpty(t, pad.Rowid) + require.Empty(t, pad.AccountData.VoteLastValid) // commit a block to get these entries removed // ensure the DB entry gone, the cache has it and lookupOnlineAccountData works as expected @@ -1071,11 +1071,11 @@ func TestAcctOnlineCacheDBSync(t *testing.T) { require.True(t, has) // full history loaded when looked up addrB prev time _, err = oa.lookupOnlineAccountData(1, addrB) require.Error(t, err) - pad, err = oa.accountsq.lookupOnline(addrB, 1) + pad, err = oa.accountsq.LookupOnline(addrB, 1) require.NoError(t, err) - require.Equal(t, addrB, pad.addr) - require.NotEmpty(t, pad.rowid) - require.NotEmpty(t, pad.accountData.VoteLastValid) + require.Equal(t, addrB, pad.Addr) + require.NotEmpty(t, pad.Rowid) + require.NotEmpty(t, pad.AccountData.VoteLastValid) }() // ensure the data not in deltas, in the cache and lookupOnlineAccountData still return a correct value @@ -1087,11 +1087,11 @@ func TestAcctOnlineCacheDBSync(t *testing.T) { data, err = oa.lookupOnlineAccountData(1, addrA) require.NoError(t, err) require.Empty(t, data.VotingData.VoteLastValid) - pad, err = oa.accountsq.lookupOnline(addrA, 1) + pad, err = oa.accountsq.LookupOnline(addrA, 1) require.NoError(t, err) - require.Equal(t, addrA, pad.addr) - require.Empty(t, pad.rowid) - require.Empty(t, pad.accountData.VoteLastValid) + require.Equal(t, addrA, pad.Addr) + require.Empty(t, pad.Rowid) + require.Empty(t, pad.AccountData.VoteLastValid) _, has = oa.accounts[addrB] require.False(t, has) @@ -1103,11 +1103,11 @@ func TestAcctOnlineCacheDBSync(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, data.VotingData.VoteLastValid) - pad, err = oa.accountsq.lookupOnline(addrB, 1) + pad, err = oa.accountsq.LookupOnline(addrB, 1) require.NoError(t, err) - require.Equal(t, addrB, pad.addr) - require.NotEmpty(t, pad.rowid) - require.NotEmpty(t, pad.accountData.VoteLastValid) + require.Equal(t, addrB, pad.Addr) + require.NotEmpty(t, pad.Rowid) + require.NotEmpty(t, pad.AccountData.VoteLastValid) }) } @@ -1190,7 +1190,7 @@ func TestAcctOnlineBaseAccountCache(t *testing.T) { commitSync(t, oa, ml, basics.Round(rnd)) poad, has := oa.baseOnlineAccounts.read(addrA) require.True(t, has) - require.Empty(t, poad.accountData) + require.Empty(t, poad.AccountData) data, err := oa.lookupOnlineAccountData(2, addrA) require.NoError(t, err) @@ -1207,7 +1207,7 @@ func TestAcctOnlineBaseAccountCache(t *testing.T) { poad, has = oa.baseOnlineAccounts.read(addrA) require.True(t, has) - require.NotEmpty(t, poad.accountData) + require.NotEmpty(t, poad.AccountData) data, err = oa.lookupOnlineAccountData(basics.Round(3), addrA) require.NoError(t, err) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 836a7c670d..9e1d294ef9 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -36,6 +36,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" @@ -156,7 +157,7 @@ type accountUpdates struct { dbs db.Pair // Prepared SQL statements for fast accounts DB lookups. - accountsq *accountsDbQueries + accountsq store.AccountsReader // cachedDBRound is always exactly tracker DB round (and therefore, accountsRound()), // cached to use in lookup functions @@ -320,7 +321,7 @@ func (au *accountUpdates) loadFromDisk(l ledgerForTracker, lastBalancesRound bas // close closes the accountUpdates, waiting for all the child go-routine to complete func (au *accountUpdates) close() { if au.accountsq != nil { - au.accountsq.close() + au.accountsq.Close() au.accountsq = nil } au.baseAccounts.prune(0) @@ -407,7 +408,7 @@ func (au *accountUpdates) lookupKv(rnd basics.Round, key string, synchronized bo // we don't technically need this, since it's already in the baseKV, however, writing this over // would ensure that we promote this field. au.baseKVs.writePending(pbd, key) - return pbd.value, nil + return pbd.Value, nil } if synchronized { @@ -419,25 +420,25 @@ func (au *accountUpdates) lookupKv(rnd basics.Round, key string, synchronized bo // roundOffset() made sure the round is exactly the one present in the // on-disk DB. - persistedData, err := au.accountsq.lookupKeyValue(key) + persistedData, err := au.accountsq.LookupKeyValue(key) if err != nil { return nil, err } - if persistedData.round == currentDbRound { + if persistedData.Round == currentDbRound { // if we read actual data return it. This includes deleted values // where persistedData.value == nil to avoid unnecessary db lookups // for deleted KVs. au.baseKVs.writePending(persistedData, key) - return persistedData.value, nil + return persistedData.Value, nil } // The db round is unexpected... if synchronized { - if persistedData.round < currentDbRound { + if persistedData.Round < currentDbRound { // Somehow the db is LOWER than it should be. - au.log.Errorf("accountUpdates.lookupKvPair: database round %d is behind in-memory round %d", persistedData.round, currentDbRound) - return nil, &StaleDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + au.log.Errorf("accountUpdates.lookupKvPair: database round %d is behind in-memory round %d", persistedData.Round, currentDbRound) + return nil, &StaleDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } // The db is higher, so a write must have happened. Try again. au.accountsMu.RLock() @@ -448,8 +449,8 @@ func (au *accountUpdates) lookupKv(rnd basics.Round, key string, synchronized bo } } else { // in non-sync mode, we don't wait since we already assume that we're synchronized. - au.log.Errorf("accountUpdates.lookupKvPair: database round %d mismatching in-memory round %d", persistedData.round, currentDbRound) - return nil, &MismatchingDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + au.log.Errorf("accountUpdates.lookupKvPair: database round %d mismatching in-memory round %d", persistedData.Round, currentDbRound) + return nil, &MismatchingDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } } @@ -552,7 +553,7 @@ func (au *accountUpdates) lookupKeysByPrefix(round basics.Round, keyPrefix strin // Finishing searching updates of this account in kvDeltas, keep going: use on-disk DB // to find the rest matching keys in DB. - dbRound, dbErr := au.accountsq.lookupKeysByPrefix(keyPrefix, maxKeyNum, results, resultCount) + dbRound, dbErr := au.accountsq.LookupKeysByPrefix(keyPrefix, maxKeyNum, results, resultCount) if dbErr != nil { return nil, dbErr } @@ -649,7 +650,7 @@ func (au *accountUpdates) listCreatables(maxCreatableIdx basics.CreatableIndex, // Fetch up to maxResults - len(res) + len(deletedCreatables) from the database, // so we have enough extras in case creatables were deleted numToFetch := maxResults - uint64(len(res)) + uint64(len(deletedCreatables)) - dbResults, dbRound, err := au.accountsq.listCreatables(maxCreatableIdx, numToFetch, ctype) + dbResults, dbRound, err := au.accountsq.ListCreatables(maxCreatableIdx, numToFetch, ctype) if err != nil { return nil, err } @@ -948,7 +949,7 @@ func (au *accountUpdates) initializeFromDisk(l ledgerForTracker, lastBalancesRou return } - au.accountsq, err = accountsInitDbQueries(au.dbs.Rdb.Handle) + au.accountsq, err = store.AccountsInitDbQueries(au.dbs.Rdb.Handle) if err != nil { return } @@ -1070,8 +1071,8 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account var offset uint64 var rewardsProto config.ConsensusParams var rewardsLevel uint64 - var persistedData persistedAccountData - var persistedResources []persistedResourcesData + var persistedData store.PersistedAccountData + var persistedResources []store.PersistedResourcesData var resourceDbRound basics.Round withRewards := true @@ -1176,11 +1177,11 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account // use a cache of the most recent account state. ad = macct.data foundAccount = true - } else if pad, has := au.baseAccounts.read(addr); has && pad.round == currentDbRound { + } else if pad, has := au.baseAccounts.read(addr); has && pad.Round == currentDbRound { // we don't technically need this, since it's already in the baseAccounts, however, writing this over // would ensure that we promote this field. au.baseAccounts.writePending(pad) - ad = pad.accountData.GetLedgerCoreAccountData() + ad = pad.AccountData.GetLedgerCoreAccountData() foundAccount = true } @@ -1205,8 +1206,8 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account // we don't technically need this, since it's already in the baseResources, however, writing this over // would ensure that we promote this field. au.baseResources.writePending(prd, addr) - if prd.addrid != 0 { - if err := addResource(prd.aidx, rnd, prd.AccountResource()); err != nil { + if prd.Addrid != 0 { + if err := addResource(prd.Aidx, rnd, prd.AccountResource()); err != nil { return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, err } } @@ -1225,15 +1226,15 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account // a separate transaction here, and directly use a prepared SQL query // against the database. if !foundAccount { - persistedData, err = au.accountsq.lookup(addr) + persistedData, err = au.accountsq.LookupAccount(addr) if err != nil { return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, err } - if persistedData.round == currentDbRound { - if persistedData.rowid != 0 { + if persistedData.Round == currentDbRound { + if persistedData.Rowid != 0 { // if we read actual data return it au.baseAccounts.writePending(persistedData) - ad = persistedData.accountData.GetLedgerCoreAccountData() + ad = persistedData.AccountData.GetLedgerCoreAccountData() } else { ad = ledgercore.AccountData{} } @@ -1244,24 +1245,24 @@ func (au *accountUpdates) lookupLatest(addr basics.Address) (data basics.Account } } - if persistedData.round < currentDbRound { - au.log.Errorf("accountUpdates.lookupLatest: account database round %d is behind in-memory round %d", persistedData.round, currentDbRound) - return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, &StaleDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + if persistedData.Round < currentDbRound { + au.log.Errorf("accountUpdates.lookupLatest: account database round %d is behind in-memory round %d", persistedData.Round, currentDbRound) + return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, &StaleDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } - if persistedData.round > currentDbRound { + if persistedData.Round > currentDbRound { goto tryAgain } } // Look for resources on disk - persistedResources, resourceDbRound, err = au.accountsq.lookupAllResources(addr) + persistedResources, resourceDbRound, err = au.accountsq.LookupAllResources(addr) if err != nil { return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, err } if resourceDbRound == currentDbRound { for _, pd := range persistedResources { au.baseResources.writePending(pd, addr) - if err := addResource(pd.aidx, currentDbRound, pd.AccountResource()); err != nil { + if err := addResource(pd.Aidx, currentDbRound, pd.AccountResource()); err != nil { return basics.AccountData{}, basics.Round(0), basics.MicroAlgos{}, err } } @@ -1295,7 +1296,7 @@ func (au *accountUpdates) lookupResource(rnd basics.Round, addr basics.Address, } }() var offset uint64 - var persistedData persistedResourcesData + var persistedData store.PersistedResourcesData for { currentDbRound := au.cachedDBRound currentDeltaLen := len(au.deltas) @@ -1355,12 +1356,12 @@ func (au *accountUpdates) lookupResource(rnd basics.Round, addr basics.Address, // present in the on-disk DB. As an optimization, we avoid creating // a separate transaction here, and directly use a prepared SQL query // against the database. - persistedData, err = au.accountsq.lookupResources(addr, aidx, ctype) + persistedData, err = au.accountsq.LookupResources(addr, aidx, ctype) if err != nil { return ledgercore.AccountResource{}, basics.Round(0), err } - if persistedData.round == currentDbRound { - if persistedData.addrid != 0 { + if persistedData.Round == currentDbRound { + if persistedData.Addrid != 0 { // if we read actual data return it au.baseResources.writePending(persistedData, addr) return persistedData.AccountResource(), rnd, nil @@ -1370,9 +1371,9 @@ func (au *accountUpdates) lookupResource(rnd basics.Round, addr basics.Address, return ledgercore.AccountResource{}, rnd, nil } if synchronized { - if persistedData.round < currentDbRound { - au.log.Errorf("accountUpdates.lookupResource: database round %d is behind in-memory round %d", persistedData.round, currentDbRound) - return ledgercore.AccountResource{}, basics.Round(0), &StaleDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + if persistedData.Round < currentDbRound { + au.log.Errorf("accountUpdates.lookupResource: database round %d is behind in-memory round %d", persistedData.Round, currentDbRound) + return ledgercore.AccountResource{}, basics.Round(0), &StaleDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } au.accountsMu.RLock() needUnlock = true @@ -1381,8 +1382,8 @@ func (au *accountUpdates) lookupResource(rnd basics.Round, addr basics.Address, } } else { // in non-sync mode, we don't wait since we already assume that we're synchronized. - au.log.Errorf("accountUpdates.lookupResource: database round %d mismatching in-memory round %d", persistedData.round, currentDbRound) - return ledgercore.AccountResource{}, basics.Round(0), &MismatchingDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + au.log.Errorf("accountUpdates.lookupResource: database round %d mismatching in-memory round %d", persistedData.Round, currentDbRound) + return ledgercore.AccountResource{}, basics.Round(0), &MismatchingDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } } } @@ -1400,7 +1401,7 @@ func (au *accountUpdates) lookupWithoutRewards(rnd basics.Round, addr basics.Add } }() var offset uint64 - var persistedData persistedAccountData + var persistedData store.PersistedAccountData for { currentDbRound := au.cachedDBRound currentDeltaLen := len(au.deltas) @@ -1445,7 +1446,7 @@ func (au *accountUpdates) lookupWithoutRewards(rnd basics.Round, addr basics.Add // we don't technically need this, since it's already in the baseAccounts, however, writing this over // would ensure that we promote this field. au.baseAccounts.writePending(macct) - return macct.accountData.GetLedgerCoreAccountData(), rnd, rewardsVersion, rewardsLevel, nil + return macct.AccountData.GetLedgerCoreAccountData(), rnd, rewardsVersion, rewardsLevel, nil } // check baseAccoiunts again to see if it does not exist @@ -1463,24 +1464,24 @@ func (au *accountUpdates) lookupWithoutRewards(rnd basics.Round, addr basics.Add // present in the on-disk DB. As an optimization, we avoid creating // a separate transaction here, and directly use a prepared SQL query // against the database. - persistedData, err = au.accountsq.lookup(addr) + persistedData, err = au.accountsq.LookupAccount(addr) if err != nil { return ledgercore.AccountData{}, basics.Round(0), "", 0, err } - if persistedData.round == currentDbRound { - if persistedData.rowid != 0 { + if persistedData.Round == currentDbRound { + if persistedData.Rowid != 0 { // if we read actual data return it au.baseAccounts.writePending(persistedData) - return persistedData.accountData.GetLedgerCoreAccountData(), rnd, rewardsVersion, rewardsLevel, nil + return persistedData.AccountData.GetLedgerCoreAccountData(), rnd, rewardsVersion, rewardsLevel, nil } au.baseAccounts.writeNotFoundPending(addr) // otherwise return empty return ledgercore.AccountData{}, rnd, rewardsVersion, rewardsLevel, nil } if synchronized { - if persistedData.round < currentDbRound { - au.log.Errorf("accountUpdates.lookupWithoutRewards: database round %d is behind in-memory round %d", persistedData.round, currentDbRound) - return ledgercore.AccountData{}, basics.Round(0), "", 0, &StaleDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + if persistedData.Round < currentDbRound { + au.log.Errorf("accountUpdates.lookupWithoutRewards: database round %d is behind in-memory round %d", persistedData.Round, currentDbRound) + return ledgercore.AccountData{}, basics.Round(0), "", 0, &StaleDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } au.accountsMu.RLock() needUnlock = true @@ -1489,8 +1490,8 @@ func (au *accountUpdates) lookupWithoutRewards(rnd basics.Round, addr basics.Add } } else { // in non-sync mode, we don't wait since we already assume that we're synchronized. - au.log.Errorf("accountUpdates.lookupWithoutRewards: database round %d mismatching in-memory round %d", persistedData.round, currentDbRound) - return ledgercore.AccountData{}, basics.Round(0), "", 0, &MismatchingDatabaseRoundError{databaseRound: persistedData.round, memoryRound: currentDbRound} + au.log.Errorf("accountUpdates.lookupWithoutRewards: database round %d mismatching in-memory round %d", persistedData.Round, currentDbRound) + return ledgercore.AccountData{}, basics.Round(0), "", 0, &MismatchingDatabaseRoundError{databaseRound: persistedData.Round, memoryRound: currentDbRound} } } } @@ -1546,7 +1547,7 @@ func (au *accountUpdates) getCreatorForRound(rnd basics.Round, cidx basics.Creat unlock = false } // Check the database - creator, ok, dbRound, err = au.accountsq.lookupCreator(cidx, ctype) + creator, ok, dbRound, err = au.accountsq.LookupCreator(cidx, ctype) if err != nil { return basics.Address{}, false, err } @@ -1675,7 +1676,7 @@ func (au *accountUpdates) commitRound(ctx context.Context, tx *sql.Tx, dcc *defe knownAddresses := make(map[basics.Address]int64, len(dcc.compactAccountDeltas.deltas)) for _, delta := range dcc.compactAccountDeltas.deltas { - knownAddresses[delta.oldAcct.addr] = delta.oldAcct.rowid + knownAddresses[delta.oldAcct.Addr] = delta.oldAcct.Rowid } err = dcc.compactResourcesDeltas.resourcesLoadOld(tx, knownAddresses) @@ -1750,14 +1751,14 @@ func (au *accountUpdates) postCommit(ctx context.Context, dcc *deferredCommitCon for i := 0; i < dcc.compactResourcesDeltas.len(); i++ { resUpdate := dcc.compactResourcesDeltas.getByIdx(i) cnt := resUpdate.nAcctDeltas - key := accountCreatable{resUpdate.address, resUpdate.oldResource.aidx} + key := accountCreatable{resUpdate.address, resUpdate.oldResource.Aidx} macct, ok := au.resources[key] if !ok { - au.log.Panicf("inconsistency: flushed %d changes to (%s, %d), but not in au.resources", cnt, resUpdate.address, resUpdate.oldResource.aidx) + au.log.Panicf("inconsistency: flushed %d changes to (%s, %d), but not in au.resources", cnt, resUpdate.address, resUpdate.oldResource.Aidx) } if cnt > macct.ndeltas { - au.log.Panicf("inconsistency: flushed %d changes to (%s, %d), but au.resources had %d", cnt, resUpdate.address, resUpdate.oldResource.aidx, macct.ndeltas) + au.log.Panicf("inconsistency: flushed %d changes to (%s, %d), but au.resources had %d", cnt, resUpdate.address, resUpdate.oldResource.Aidx, macct.ndeltas) } else if cnt == macct.ndeltas { delete(au.resources, key) } else { diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index a19217fed4..f6e954d7e0 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -38,6 +38,7 @@ import ( "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger/internal" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -1198,7 +1199,7 @@ func TestListCreatables(t *testing.T) { require.NoError(t, err) au := &accountUpdates{} - au.accountsq, err = accountsInitDbQueries(tx) + au.accountsq, err = store.AccountsInitDbQueries(tx) require.NoError(t, err) // ******* All results are obtained from the cache. Empty database ******* @@ -1435,7 +1436,7 @@ func TestKVCache(t *testing.T) { name := fmt.Sprintf("%d", uint64(startKV)+uint64(j)) persistedValue, has := au.baseKVs.read(name) require.True(t, has) - require.Equal(t, kvMap[name], persistedValue.value) + require.Equal(t, kvMap[name], persistedValue.Value) } } } @@ -1468,7 +1469,7 @@ func TestKVCache(t *testing.T) { for name := range kvMods { persistedValue, has := au.baseKVs.read(name) require.True(t, has) - require.Equal(t, kvMap[name], persistedValue.value) + require.Equal(t, kvMap[name], persistedValue.Value) } } @@ -1485,7 +1486,7 @@ func TestKVCache(t *testing.T) { persistedValue, has := au.baseKVs.read(name) require.True(t, has) expectedValue := fmt.Sprintf("modified value%s", name) - require.Equal(t, expectedValue, string(persistedValue.value)) + require.Equal(t, expectedValue, string(persistedValue.Value)) } } } @@ -1520,7 +1521,7 @@ func TestKVCache(t *testing.T) { persistedValue, has := au.baseKVs.read(name) require.True(t, has) value := fmt.Sprintf("modified value%s", name) - require.Equal(t, value, string(persistedValue.value)) + require.Equal(t, value, string(persistedValue.Value)) } } @@ -1536,7 +1537,7 @@ func TestKVCache(t *testing.T) { name := fmt.Sprintf("%d", uint64(startKV)+uint64(j)) persistedValue, has := au.baseKVs.read(name) require.True(t, has) - require.True(t, persistedValue.value == nil) + require.True(t, persistedValue.Value == nil) } } } @@ -1559,7 +1560,7 @@ func accountsAll(tx *sql.Tx) (bals map[basics.Address]basics.AccountData, err er return } - var data baseAccountData + var data store.BaseAccountData err = protocol.Decode(buf, &data) if err != nil { return @@ -1614,7 +1615,7 @@ func BenchmarkLargeMerkleTrieRebuild(b *testing.B) { var updates compactAccountDeltas for k := 0; i < accountsNumber-5-2 && k < 1024; k++ { addr := ledgertesting.RandomAddress() - acctData := baseAccountData{} + acctData := store.BaseAccountData{} acctData.MicroAlgos.Raw = 1 updates.upsert(addr, accountDelta{newAcct: acctData}) i++ @@ -1695,17 +1696,17 @@ func TestCompactDeltas(t *testing.T) { // check deltas with missing accounts delta, _ := outAccountDeltas.get(addrs[0]) - require.Equal(t, persistedAccountData{}, delta.oldAcct) + require.Equal(t, store.PersistedAccountData{}, delta.oldAcct) require.NotEmpty(t, delta.newAcct) require.Equal(t, ledgercore.ModifiedCreatable{Creator: addrs[2], Created: true, Ndeltas: 1}, outCreatableDeltas[100]) // check deltas without missing accounts - baseAccounts.write(persistedAccountData{addr: addrs[0], accountData: baseAccountData{}}) + baseAccounts.write(store.PersistedAccountData{Addr: addrs[0], AccountData: store.BaseAccountData{}}) outAccountDeltas = makeCompactAccountDeltas(accountDeltas, basics.Round(1), true, baseAccounts) require.Equal(t, 0, len(outAccountDeltas.misses)) delta, _ = outAccountDeltas.get(addrs[0]) - require.Equal(t, persistedAccountData{addr: addrs[0]}, delta.oldAcct) - require.Equal(t, baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 2}, UpdateRound: 2}, delta.newAcct) + require.Equal(t, store.PersistedAccountData{Addr: addrs[0]}, delta.oldAcct) + require.Equal(t, store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 2}, UpdateRound: 2}, delta.newAcct) require.Equal(t, ledgercore.ModifiedCreatable{Creator: addrs[2], Created: true, Ndeltas: 1}, outCreatableDeltas[100]) baseAccounts.init(nil, 100, 80) @@ -1718,8 +1719,8 @@ func TestCompactDeltas(t *testing.T) { creatableDeltas[1][100] = ledgercore.ModifiedCreatable{Creator: addrs[2], Created: false} creatableDeltas[1][101] = ledgercore.ModifiedCreatable{Creator: addrs[4], Created: true} - baseAccounts.write(persistedAccountData{addr: addrs[0], accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 1}}}) - baseAccounts.write(persistedAccountData{addr: addrs[3], accountData: baseAccountData{}}) + baseAccounts.write(store.PersistedAccountData{Addr: addrs[0], AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: 1}}}) + baseAccounts.write(store.PersistedAccountData{Addr: addrs[3], AccountData: store.BaseAccountData{}}) outAccountDeltas = makeCompactAccountDeltas(accountDeltas, basics.Round(1), true, baseAccounts) outCreatableDeltas = compactCreatableDeltas(creatableDeltas) @@ -1727,11 +1728,11 @@ func TestCompactDeltas(t *testing.T) { require.Equal(t, 2, len(outCreatableDeltas)) delta, _ = outAccountDeltas.get(addrs[0]) - require.Equal(t, uint64(1), delta.oldAcct.accountData.MicroAlgos.Raw) + require.Equal(t, uint64(1), delta.oldAcct.AccountData.MicroAlgos.Raw) require.Equal(t, uint64(3), delta.newAcct.MicroAlgos.Raw) require.Equal(t, int(2), delta.nAcctDeltas) delta, _ = outAccountDeltas.get(addrs[3]) - require.Equal(t, uint64(0), delta.oldAcct.accountData.MicroAlgos.Raw) + require.Equal(t, uint64(0), delta.oldAcct.AccountData.MicroAlgos.Raw) require.Equal(t, uint64(8), delta.newAcct.MicroAlgos.Raw) require.Equal(t, int(1), delta.nAcctDeltas) @@ -1766,22 +1767,22 @@ func TestCompactDeltasResources(t *testing.T) { delta, _ := outResourcesDeltas.get(addrs[0], 100) require.NotEmpty(t, delta.newResource) require.True(t, !delta.newResource.IsApp() && !delta.newResource.IsAsset()) - require.Equal(t, resourceFlagsNotHolding, delta.newResource.ResourceFlags) + require.Equal(t, store.ResourceFlagsNotHolding, delta.newResource.ResourceFlags) delta, _ = outResourcesDeltas.get(addrs[1], 101) require.NotEmpty(t, delta.newResource) require.True(t, !delta.newResource.IsApp() && !delta.newResource.IsAsset()) - require.Equal(t, resourceFlagsNotHolding, delta.newResource.ResourceFlags) + require.Equal(t, store.ResourceFlagsNotHolding, delta.newResource.ResourceFlags) delta, _ = outResourcesDeltas.get(addrs[2], 102) require.NotEmpty(t, delta.newResource) require.True(t, !delta.newResource.IsApp() && !delta.newResource.IsAsset()) - require.Equal(t, resourceFlagsNotHolding, delta.newResource.ResourceFlags) + require.Equal(t, store.ResourceFlagsNotHolding, delta.newResource.ResourceFlags) delta, _ = outResourcesDeltas.get(addrs[3], 103) require.NotEmpty(t, delta.newResource) require.True(t, !delta.newResource.IsApp() && !delta.newResource.IsAsset()) - require.Equal(t, resourceFlagsNotHolding, delta.newResource.ResourceFlags) + require.Equal(t, store.ResourceFlagsNotHolding, delta.newResource.ResourceFlags) // check actual data on non-empty input accountDeltas = make([]ledgercore.AccountDeltas, 1) @@ -1853,19 +1854,19 @@ func TestCompactDeltasResources(t *testing.T) { for i := int64(0); i < 4; i++ { delta, idx := outResourcesDeltas.get(addrs[i], basics.CreatableIndex(100+i)) require.NotEqual(t, -1, idx) - require.Equal(t, persistedResourcesData{aidx: basics.CreatableIndex(100 + i)}, delta.oldResource) + require.Equal(t, store.PersistedResourcesData{Aidx: basics.CreatableIndex(100 + i)}, delta.oldResource) if i%2 == 0 { delta, idx = outResourcesDeltas.get(addrs[i], basics.CreatableIndex(200+i)) require.NotEqual(t, -1, idx) - require.Equal(t, persistedResourcesData{aidx: basics.CreatableIndex(200 + i)}, delta.oldResource) + require.Equal(t, store.PersistedResourcesData{Aidx: basics.CreatableIndex(200 + i)}, delta.oldResource) } } // check deltas without missing accounts for i := int64(0); i < 4; i++ { - baseResources.write(persistedResourcesData{addrid: i + 1, aidx: basics.CreatableIndex(100 + i)}, addrs[i]) + baseResources.write(store.PersistedResourcesData{Addrid: i + 1, Aidx: basics.CreatableIndex(100 + i)}, addrs[i]) if i%2 == 0 { - baseResources.write(persistedResourcesData{addrid: i + 1, aidx: basics.CreatableIndex(200 + i)}, addrs[i]) + baseResources.write(store.PersistedResourcesData{Addrid: i + 1, Aidx: basics.CreatableIndex(200 + i)}, addrs[i]) } } @@ -1877,11 +1878,11 @@ func TestCompactDeltasResources(t *testing.T) { for i := int64(0); i < 4; i++ { delta, idx := outResourcesDeltas.get(addrs[i], basics.CreatableIndex(100+i)) require.NotEqual(t, -1, idx) - require.Equal(t, persistedResourcesData{addrid: i + 1, aidx: basics.CreatableIndex(100 + i)}, delta.oldResource) + require.Equal(t, store.PersistedResourcesData{Addrid: i + 1, Aidx: basics.CreatableIndex(100 + i)}, delta.oldResource) if i%2 == 0 { delta, idx = outResourcesDeltas.get(addrs[i], basics.CreatableIndex(200+i)) require.NotEqual(t, -1, idx) - require.Equal(t, persistedResourcesData{addrid: i + 1, aidx: basics.CreatableIndex(200 + i)}, delta.oldResource) + require.Equal(t, store.PersistedResourcesData{Addrid: i + 1, Aidx: basics.CreatableIndex(200 + i)}, delta.oldResource) } } @@ -1897,7 +1898,7 @@ func TestCompactDeltasResources(t *testing.T) { appLocalState204 := basics.AppLocalState{KeyValue: basics.TealKeyValue{"204": basics.TealValue{Type: basics.TealBytesType, Bytes: "204"}}} accountDeltas[1].UpsertAppResource(addrs[4], 104, ledgercore.AppParamsDelta{Params: &appParams104}, ledgercore.AppLocalStateDelta{LocalState: &appLocalState204}) - baseResources.write(persistedResourcesData{addrid: 5 /* 4+1 */, aidx: basics.CreatableIndex(104)}, addrs[4]) + baseResources.write(store.PersistedResourcesData{Addrid: 5 /* 4+1 */, Aidx: basics.CreatableIndex(104)}, addrs[4]) outResourcesDeltas = makeCompactResourceDeltas(accountDeltas, basics.Round(1), true, baseAccounts, baseResources) require.Equal(t, 0, len(outResourcesDeltas.misses)) @@ -2856,16 +2857,16 @@ func TestAcctUpdatesLookupLatestCacheRetry(t *testing.T) { pad, ok := au.baseAccounts.read(addr1) require.True(t, ok) - pad.round = au.cachedDBRound + pad.Round = au.cachedDBRound au.baseAccounts.write(pad) prd, ok := au.baseResources.read(addr1, basics.CreatableIndex(aidx1)) require.True(t, ok) - prd.round = oldCachedDBRound + prd.Round = oldCachedDBRound au.baseResources.write(prd, addr1) prd, ok = au.baseResources.read(addr1, basics.CreatableIndex(aidx2)) require.True(t, ok) - prd.round = oldCachedDBRound + prd.Round = oldCachedDBRound au.baseResources.write(prd, addr1) var ad basics.AccountData diff --git a/ledger/applications_test.go b/ledger/applications_test.go index 012e291b4e..67dbfd50be 100644 --- a/ledger/applications_test.go +++ b/ledger/applications_test.go @@ -27,6 +27,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" + "github.com/algorand/go-algorand/ledger/store" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -250,48 +251,45 @@ return` commitRound(1, 3, l) // dump accounts - var rowid int64 - var dbRound basics.Round - var buf []byte - err = l.accts.accountsq.lookupStmt.QueryRow(creator[:]).Scan(&rowid, &dbRound, &buf) + entryAcc, err := l.accts.accountsq.LookupAccount(creator) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedCreatorBase, buf) - err = l.accts.accountsq.lookupResourcesStmt.QueryRow(creator[:], basics.CreatableIndex(appIdx)).Scan(&rowid, &dbRound, &buf) + a.Equal(basics.Round(4), entryAcc.Round) + a.Equal(expectedCreatorBase, protocol.Encode(&entryAcc.AccountData)) + entryRes, err := l.accts.accountsq.LookupResources(creator, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedCreatorResource, buf) + a.Equal(basics.Round(4), entryRes.Round) + a.Equal(expectedCreatorResource, protocol.Encode(&entryRes.Data)) - err = l.accts.accountsq.lookupStmt.QueryRow(userOptin[:]).Scan(&rowid, &dbRound, &buf) + entryAcc, err = l.accts.accountsq.LookupAccount(userOptin) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedUserOptInBase, buf) - err = l.accts.accountsq.lookupResourcesStmt.QueryRow(userOptin[:], basics.CreatableIndex(appIdx)).Scan(&rowid, &dbRound, &buf) + a.Equal(basics.Round(4), entryAcc.Round) + a.Equal(expectedUserOptInBase, protocol.Encode(&entryAcc.AccountData)) + entryRes, err = l.accts.accountsq.LookupResources(userOptin, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedUserOptInResource, buf) + a.Equal(basics.Round(4), entryRes.Round) + a.Equal(expectedUserOptInResource, protocol.Encode(&entryRes.Data)) - pad, err := l.accts.accountsq.lookup(userOptin) + pad, err := l.accts.accountsq.LookupAccount(userOptin) a.NoError(err) a.NotEmpty(pad) - prd, err := l.accts.accountsq.lookupResources(userOptin, basics.CreatableIndex(appIdx), basics.AppCreatable) + prd, err := l.accts.accountsq.LookupResources(userOptin, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Nil(prd.data.GetAppLocalState().KeyValue) + a.Nil(prd.Data.GetAppLocalState().KeyValue) ad, rnd, _, err := l.LookupLatest(userOptin) - a.Equal(dbRound, rnd) + a.Equal(basics.Round(4), rnd) a.NoError(err) a.Nil(ad.AppLocalStates[appIdx].KeyValue) - err = l.accts.accountsq.lookupStmt.QueryRow(userLocal[:]).Scan(&rowid, &dbRound, &buf) + entryAcc, err = l.accts.accountsq.LookupAccount(userLocal) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedUserLocalBase, buf) - err = l.accts.accountsq.lookupResourcesStmt.QueryRow(userLocal[:], basics.CreatableIndex(appIdx)).Scan(&rowid, &dbRound, &buf) + a.Equal(basics.Round(4), entryAcc.Round) + a.Equal(expectedUserLocalBase, protocol.Encode(&entryAcc.AccountData)) + entryRes, err = l.accts.accountsq.LookupResources(userLocal, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Equal(basics.Round(4), dbRound) - a.Equal(expectedUserLocalResource, buf) + a.Equal(basics.Round(4), entryRes.Round) + a.Equal(expectedUserLocalResource, protocol.Encode(&entryRes.Data)) - ar, err := l.LookupApplication(dbRound, userLocal, appIdx) + ar, err := l.LookupApplication(basics.Round(4), userLocal, appIdx) a.NoError(err) a.Equal("local", ar.AppLocalState.KeyValue["lk"].Bytes) @@ -753,15 +751,15 @@ return` a.NoError(err) a.Empty(blk.Payset[0].ApplyData.EvalDelta.LocalDeltas) - pad, err := l.accts.accountsq.lookup(userLocal) + pad, err := l.accts.accountsq.LookupAccount(userLocal) a.NoError(err) - a.Equal(baseAccountData{}, pad.accountData) - a.Zero(pad.rowid) - prd, err := l.accts.accountsq.lookupResources(userLocal, basics.CreatableIndex(appIdx), basics.AppCreatable) + a.Equal(store.BaseAccountData{}, pad.AccountData) + a.Zero(pad.Rowid) + prd, err := l.accts.accountsq.LookupResources(userLocal, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Zero(prd.addrid) - emptyResourceData := makeResourcesData(0) - a.Equal(emptyResourceData, prd.data) + a.Zero(prd.Addrid) + emptyResourceData := store.MakeResourcesData(0) + a.Equal(emptyResourceData, prd.Data) } func TestAppEmptyAccountsGlobal(t *testing.T) { @@ -889,15 +887,15 @@ return` a.Contains(blk.Payset[0].ApplyData.EvalDelta.GlobalDelta, "gk") a.Equal(blk.Payset[0].ApplyData.EvalDelta.GlobalDelta["gk"].Bytes, "global") - pad, err := l.accts.accountsq.lookup(creator) + pad, err := l.accts.accountsq.LookupAccount(creator) a.NoError(err) - a.Empty(pad.accountData) - a.Zero(pad.rowid) - prd, err := l.accts.accountsq.lookupResources(creator, basics.CreatableIndex(appIdx), basics.AppCreatable) + a.Empty(pad.AccountData) + a.Zero(pad.Rowid) + prd, err := l.accts.accountsq.LookupResources(creator, basics.CreatableIndex(appIdx), basics.AppCreatable) a.NoError(err) - a.Zero(prd.addrid) - emptyResourceData := makeResourcesData(0) - a.Equal(emptyResourceData, prd.data) + a.Zero(prd.Addrid) + emptyResourceData := store.MakeResourcesData(0) + a.Equal(emptyResourceData, prd.Data) } func TestAppAccountDeltaIndicesCompatibility1(t *testing.T) { diff --git a/ledger/blockdb_test.go b/ledger/blockdb_test.go index da1d8303c9..39cf6eaa0a 100644 --- a/ledger/blockdb_test.go +++ b/ledger/blockdb_test.go @@ -18,8 +18,6 @@ package ledger import ( "database/sql" - "fmt" - "strings" "testing" "github.com/stretchr/testify/require" @@ -28,17 +26,20 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/logging" + storetesting "github.com/algorand/go-algorand/ledger/store/testing" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/db" ) +// TODO[store-refactor]: temporary leftovers, refactor calls to store.* func dbOpenTest(t testing.TB, inMemory bool) (db.Pair, string) { - fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) - dbs, err := db.OpenPair(fn, inMemory) - require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory) - return dbs, fn + return storetesting.DbOpenTest(t, inMemory) +} + +// TODO[store-refactor]: temporary leftovers, refactor calls to store.* +func setDbLogging(t testing.TB, dbs db.Pair) { + storetesting.SetDbLogging(t, dbs) } func randomBlock(r basics.Round) blockEntry { @@ -112,12 +113,6 @@ func checkBlockDB(t *testing.T, tx *sql.Tx, blocks []blockEntry) { require.Error(t, err) } -func setDbLogging(t testing.TB, dbs db.Pair) { - dblogger := logging.TestingLog(t) - dbs.Rdb.SetLogger(dblogger) - dbs.Wdb.SetLogger(dblogger) -} - func TestBlockDBEmpty(t *testing.T) { partitiontest.PartitionTest(t) diff --git a/ledger/catchpointtracker.go b/ledger/catchpointtracker.go index 584a9447d8..d890584332 100644 --- a/ledger/catchpointtracker.go +++ b/ledger/catchpointtracker.go @@ -43,6 +43,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" @@ -111,7 +112,7 @@ type catchpointTracker struct { enableGeneratingCatchpointFiles bool // Prepared SQL statements for fast accounts DB lookups. - accountsq *accountsDbQueries + accountsq store.AccountsReader // log copied from ledger log logging.Logger @@ -345,7 +346,7 @@ func (ct *catchpointTracker) loadFromDisk(l ledgerForTracker, dbRound basics.Rou return err } - ct.accountsq, err = accountsInitDbQueries(ct.dbs.Rdb.Handle) + ct.accountsq, err = store.AccountsInitDbQueries(ct.dbs.Rdb.Handle) if err != nil { return } @@ -940,8 +941,8 @@ func (ct *catchpointTracker) accountsUpdateBalances(accountsDeltas compactAccoun for i := 0; i < accountsDeltas.len(); i++ { delta := accountsDeltas.getByIdx(i) - if !delta.oldAcct.accountData.IsEmpty() { - deleteHash := accountHashBuilderV6(delta.address, &delta.oldAcct.accountData, protocol.Encode(&delta.oldAcct.accountData)) + if !delta.oldAcct.AccountData.IsEmpty() { + deleteHash := accountHashBuilderV6(delta.address, &delta.oldAcct.AccountData, protocol.Encode(&delta.oldAcct.AccountData)) deleted, err = ct.balancesTrie.Delete(deleteHash) if err != nil { return fmt.Errorf("failed to delete hash '%s' from merkle trie for account %v: %w", hex.EncodeToString(deleteHash), delta.address, err) @@ -970,8 +971,8 @@ func (ct *catchpointTracker) accountsUpdateBalances(accountsDeltas compactAccoun for i := 0; i < resourcesDeltas.len(); i++ { resDelta := resourcesDeltas.getByIdx(i) addr := resDelta.address - if !resDelta.oldResource.data.IsEmpty() { - deleteHash, err := resourcesHashBuilderV6(&resDelta.oldResource.data, addr, resDelta.oldResource.aidx, resDelta.oldResource.data.UpdateRound, protocol.Encode(&resDelta.oldResource.data)) + if !resDelta.oldResource.Data.IsEmpty() { + deleteHash, err := resourcesHashBuilderV6(&resDelta.oldResource.Data, addr, resDelta.oldResource.Aidx, resDelta.oldResource.Data.UpdateRound, protocol.Encode(&resDelta.oldResource.Data)) if err != nil { return err } @@ -987,7 +988,7 @@ func (ct *catchpointTracker) accountsUpdateBalances(accountsDeltas compactAccoun } if !resDelta.newResource.IsEmpty() { - addHash, err := resourcesHashBuilderV6(&resDelta.newResource, addr, resDelta.oldResource.aidx, resDelta.newResource.UpdateRound, protocol.Encode(&resDelta.newResource)) + addHash, err := resourcesHashBuilderV6(&resDelta.newResource, addr, resDelta.oldResource.Aidx, resDelta.newResource.UpdateRound, protocol.Encode(&resDelta.newResource)) if err != nil { return err } @@ -1426,7 +1427,7 @@ func finishV6(v6hash []byte, prehash []byte) []byte { } // accountHashBuilderV6 calculates the hash key used for the trie by combining the account address and the account data -func accountHashBuilderV6(addr basics.Address, accountData *baseAccountData, encodedAccountData []byte) []byte { +func accountHashBuilderV6(addr basics.Address, accountData *store.BaseAccountData, encodedAccountData []byte) []byte { hashIntPrefix := accountData.UpdateRound if hashIntPrefix == 0 { hashIntPrefix = accountData.RewardsBase @@ -1446,6 +1447,7 @@ func accountHashBuilderV6(addr basics.Address, accountData *baseAccountData, enc // trie. Each merkle trie hash includes the hashKind byte at a known-offset. // By encoding hashKind at a known-offset, it's possible for hash readers to // disambiguate the hashed resource. +// //go:generate stringer -type=hashKind type hashKind byte @@ -1462,7 +1464,7 @@ const ( // encoded. const hashKindEncodingIndex = 4 -func rdGetCreatableHashKind(rd *resourcesData, a basics.Address, ci basics.CreatableIndex) (hashKind, error) { +func rdGetCreatableHashKind(rd *store.ResourcesData, a basics.Address, ci basics.CreatableIndex) (hashKind, error) { if rd.IsAsset() { return assetHK, nil } else if rd.IsApp() { @@ -1472,7 +1474,7 @@ func rdGetCreatableHashKind(rd *resourcesData, a basics.Address, ci basics.Creat } // resourcesHashBuilderV6 calculates the hash key used for the trie by combining the creatable's resource data and its index -func resourcesHashBuilderV6(rd *resourcesData, addr basics.Address, cidx basics.CreatableIndex, updateRound uint64, encodedResourceData []byte) ([]byte, error) { +func resourcesHashBuilderV6(rd *store.ResourcesData, addr basics.Address, cidx basics.CreatableIndex, updateRound uint64, encodedResourceData []byte) ([]byte, error) { hk, err := rdGetCreatableHashKind(rd, addr, cidx) if err != nil { return nil, err diff --git a/ledger/catchpointtracker_test.go b/ledger/catchpointtracker_test.go index 7e9b4d19a4..ef636dcf94 100644 --- a/ledger/catchpointtracker_test.go +++ b/ledger/catchpointtracker_test.go @@ -39,6 +39,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -357,7 +358,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) { var updates compactAccountDeltas for k := 0; i < accountsNumber-5-2 && k < 1024; k++ { addr := ledgertesting.RandomAddress() - acctData := baseAccountData{} + acctData := store.BaseAccountData{} acctData.MicroAlgos.Raw = 1 updates.upsert(addr, accountDelta{newAcct: acctData}) i++ @@ -1391,7 +1392,7 @@ func TestHashContract(t *testing.T) { accounts := []testCase{ accountCase( func() []byte { - b := baseAccountData{ + b := store.BaseAccountData{ UpdateRound: 1024, } return accountHashBuilderV6(a, &b, protocol.Encode(&b)) @@ -1400,7 +1401,7 @@ func TestHashContract(t *testing.T) { ), accountCase( func() []byte { - b := baseAccountData{ + b := store.BaseAccountData{ RewardsBase: 10000, } return accountHashBuilderV6(a, &b, protocol.Encode(&b)) @@ -1412,7 +1413,7 @@ func TestHashContract(t *testing.T) { resourceAssets := []testCase{ resourceAssetCase( func() []byte { - r := resourcesData{ + r := store.ResourcesData{ Amount: 1000, Decimals: 3, AssetName: "test", @@ -1430,7 +1431,7 @@ func TestHashContract(t *testing.T) { resourceApps := []testCase{ resourceAppCase( func() []byte { - r := resourcesData{ + r := store.ResourcesData{ ApprovalProgram: []byte{1, 3, 10, 15}, ClearStateProgram: []byte{15, 10, 3, 1}, LocalStateSchemaNumUint: 2, diff --git a/ledger/catchupaccessor_test.go b/ledger/catchupaccessor_test.go index b95a924ce3..f6283f9b70 100644 --- a/ledger/catchupaccessor_test.go +++ b/ledger/catchupaccessor_test.go @@ -33,6 +33,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -59,7 +60,7 @@ func createTestingEncodedChunks(accountsCount uint64) (encodedAccountChunks [][] chunk.Balances = make([]encodedBalanceRecordV6, chunkSize) for i := uint64(0); i < chunkSize; i++ { var randomAccount encodedBalanceRecordV6 - accountData := baseAccountData{} + accountData := store.BaseAccountData{} accountData.MicroAlgos.Raw = crypto.RandUint63() randomAccount.AccountData = protocol.Encode(&accountData) // have the first account be the zero address @@ -407,7 +408,7 @@ func TestCatchupAccessorResourceCountMismatch(t *testing.T) { var balances catchpointFileChunkV6 balances.Balances = make([]encodedBalanceRecordV6, 1) var randomAccount encodedBalanceRecordV6 - accountData := baseAccountData{} + accountData := store.BaseAccountData{} accountData.MicroAlgos.Raw = crypto.RandUint63() accountData.TotalAppParams = 1 randomAccount.AccountData = protocol.Encode(&accountData) @@ -475,8 +476,8 @@ func TestCatchupAccessorProcessStagingBalances(t *testing.T) { } catchpointAccessor := makeTestCatchpointCatchupAccessor(&l, log, writer) - randomSimpleBaseAcct := func() baseAccountData { - accountData := baseAccountData{ + randomSimpleBaseAcct := func() store.BaseAccountData { + accountData := store.BaseAccountData{ RewardsBase: crypto.RandUint63(), MicroAlgos: basics.MicroAlgos{Raw: crypto.RandUint63()}, AuthAddr: ledgertesting.RandomAddress(), @@ -484,7 +485,7 @@ func TestCatchupAccessorProcessStagingBalances(t *testing.T) { return accountData } - encodedBalanceRecordFromBase := func(addr basics.Address, base baseAccountData, resources map[uint64]msgp.Raw, more bool) encodedBalanceRecordV6 { + encodedBalanceRecordFromBase := func(addr basics.Address, base store.BaseAccountData, resources map[uint64]msgp.Raw, more bool) encodedBalanceRecordV6 { ebr := encodedBalanceRecordV6{ Address: addr, AccountData: protocol.Encode(&base), @@ -516,7 +517,7 @@ func TestCatchupAccessorProcessStagingBalances(t *testing.T) { acctX.TotalAssets = acctXNumRes acctXRes1 := make(map[uint64]msgp.Raw, acctXNumRes/2+1) acctXRes2 := make(map[uint64]msgp.Raw, acctXNumRes/2) - emptyRes := resourcesData{ResourceFlags: resourceFlagsEmptyAsset} + emptyRes := store.ResourcesData{ResourceFlags: store.ResourceFlagsEmptyAsset} emptyResEnc := protocol.Encode(&emptyRes) for i := 0; i < acctXNumRes; i++ { if i <= acctXNumRes/2 { diff --git a/ledger/lruaccts.go b/ledger/lruaccts.go index f698f9de76..13e639a95a 100644 --- a/ledger/lruaccts.go +++ b/ledger/lruaccts.go @@ -18,6 +18,7 @@ package ledger import ( "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" ) @@ -32,7 +33,7 @@ type lruAccounts struct { accounts map[basics.Address]*persistedAccountDataListNode // pendingAccounts are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these, // it would call flushPendingWrites and these would be merged into the accounts/accountsList - pendingAccounts chan persistedAccountData + pendingAccounts chan store.PersistedAccountData // log interface; used for logging the threshold event. log logging.Logger // pendingWritesWarnThreshold is the threshold beyond we would write a warning for exceeding the number of pendingAccounts entries @@ -47,7 +48,7 @@ type lruAccounts struct { func (m *lruAccounts) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) { m.accountsList = newPersistedAccountList().allocateFreeNodes(pendingWrites) m.accounts = make(map[basics.Address]*persistedAccountDataListNode, pendingWrites) - m.pendingAccounts = make(chan persistedAccountData, pendingWrites) + m.pendingAccounts = make(chan store.PersistedAccountData, pendingWrites) m.notFound = make(map[basics.Address]struct{}, pendingWrites) m.pendingNotFound = make(chan basics.Address, pendingWrites) m.log = log @@ -56,11 +57,11 @@ func (m *lruAccounts) init(log logging.Logger, pendingWrites int, pendingWritesW // read the persistedAccountData object that the lruAccounts has for the given address. // thread locking semantics : read lock -func (m *lruAccounts) read(addr basics.Address) (data persistedAccountData, has bool) { +func (m *lruAccounts) read(addr basics.Address) (data store.PersistedAccountData, has bool) { if el := m.accounts[addr]; el != nil { return *el.Value, true } - return persistedAccountData{}, false + return store.PersistedAccountData{}, false } // readNotFound returns whether we have attempted to read this address but it did not exist in the db. @@ -103,7 +104,7 @@ outer2: // writePending write a single persistedAccountData entry to the pendingAccounts buffer. // the function doesn't block, and in case of a buffer overflow the entry would not be added. // thread locking semantics : no lock is required. -func (m *lruAccounts) writePending(acct persistedAccountData) { +func (m *lruAccounts) writePending(acct store.PersistedAccountData) { select { case m.pendingAccounts <- acct: default: @@ -125,17 +126,17 @@ func (m *lruAccounts) writeNotFoundPending(addr basics.Address) { // version of what's already on the cache or not. In all cases, the entry is going // to be promoted to the front of the list. // thread locking semantics : write lock -func (m *lruAccounts) write(acctData persistedAccountData) { - if el := m.accounts[acctData.addr]; el != nil { +func (m *lruAccounts) write(acctData store.PersistedAccountData) { + if el := m.accounts[acctData.Addr]; el != nil { // already exists; is it a newer ? - if el.Value.before(&acctData) { + if el.Value.Before(&acctData) { // we update with a newer version. el.Value = &acctData } m.accountsList.moveToFront(el) } else { // new entry. - m.accounts[acctData.addr] = m.accountsList.pushFront(&acctData) + m.accounts[acctData.Addr] = m.accountsList.pushFront(&acctData) } } @@ -148,7 +149,7 @@ func (m *lruAccounts) prune(newSize int) (removed int) { break } back := m.accountsList.back() - delete(m.accounts, back.Value.addr) + delete(m.accounts, back.Value.Addr) m.accountsList.remove(back) removed++ } diff --git a/ledger/lruaccts_test.go b/ledger/lruaccts_test.go index a5f8509013..30f1f082c6 100644 --- a/ledger/lruaccts_test.go +++ b/ledger/lruaccts_test.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -38,11 +39,11 @@ func TestLRUBasicAccounts(t *testing.T) { accountsNum := 50 // write 50 accounts for i := 0; i < accountsNum; i++ { - acct := persistedAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseAcct.write(acct) } @@ -52,10 +53,10 @@ func TestLRUBasicAccounts(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseAcct.read(addr) require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } // verify expected missing entries @@ -63,7 +64,7 @@ func TestLRUBasicAccounts(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseAcct.read(addr) require.False(t, has) - require.Equal(t, persistedAccountData{}, acct) + require.Equal(t, store.PersistedAccountData{}, acct) } baseAcct.prune(accountsNum / 2) @@ -76,13 +77,13 @@ func TestLRUBasicAccounts(t *testing.T) { if i >= accountsNum/2 && i < accountsNum { // expected to have it. require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } else { require.False(t, has) - require.Equal(t, persistedAccountData{}, acct) + require.Equal(t, store.PersistedAccountData{}, acct) } } } @@ -97,11 +98,11 @@ func TestLRUAccountsPendingWrites(t *testing.T) { for i := 0; i < accountsNum; i++ { go func(i int) { time.Sleep(time.Duration((crypto.RandUint64() % 50)) * time.Millisecond) - acct := persistedAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseAcct.writePending(acct) }(i) @@ -149,11 +150,11 @@ func TestLRUAccountsPendingWritesWarning(t *testing.T) { baseAcct.init(log, pendingWritesBuffer, pendingWritesThreshold) for j := 0; j < 50; j++ { for i := 0; i < j; i++ { - acct := persistedAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseAcct.writePending(acct) } @@ -175,11 +176,11 @@ func TestLRUAccountsOmittedPendingWrites(t *testing.T) { baseAcct.init(log, pendingWritesBuffer, pendingWritesThreshold) for i := 0; i < pendingWritesBuffer*2; i++ { - acct := persistedAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseAcct.writePending(acct) } @@ -191,10 +192,10 @@ func TestLRUAccountsOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseAcct.read(addr) require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } // verify expected missing entries @@ -202,7 +203,7 @@ func TestLRUAccountsOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseAcct.read(addr) require.False(t, has) - require.Equal(t, persistedAccountData{}, acct) + require.Equal(t, store.PersistedAccountData{}, acct) } } @@ -215,7 +216,7 @@ func BenchmarkLRUAccountsWrite(b *testing.B) { benchLruWrite(b, fillerAccounts, accounts) } -func benchLruWrite(b *testing.B, fillerAccounts []persistedAccountData, accounts []persistedAccountData) { +func benchLruWrite(b *testing.B, fillerAccounts []store.PersistedAccountData, accounts []store.PersistedAccountData) { b.ResetTimer() b.StopTimer() var baseAcct lruAccounts @@ -231,26 +232,26 @@ func benchLruWrite(b *testing.B, fillerAccounts []persistedAccountData, accounts } } -func fillLRUAccounts(baseAcct lruAccounts, fillerAccounts []persistedAccountData) lruAccounts { +func fillLRUAccounts(baseAcct lruAccounts, fillerAccounts []store.PersistedAccountData) lruAccounts { for _, account := range fillerAccounts { baseAcct.write(account) } return baseAcct } -func generatePersistedAccountData(startRound, endRound int) []persistedAccountData { - accounts := make([]persistedAccountData, endRound-startRound) +func generatePersistedAccountData(startRound, endRound int) []store.PersistedAccountData { + accounts := make([]store.PersistedAccountData, endRound-startRound) buffer := make([]byte, 4) for i := startRound; i < endRound; i++ { binary.BigEndian.PutUint32(buffer, uint32(i)) digest := crypto.Hash(buffer) - accounts[i-startRound] = persistedAccountData{ - addr: basics.Address(digest), - round: basics.Round(i + startRound), - rowid: int64(i), - accountData: baseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + accounts[i-startRound] = store.PersistedAccountData{ + Addr: basics.Address(digest), + Round: basics.Round(i + startRound), + Rowid: int64(i), + AccountData: store.BaseAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } } return accounts diff --git a/ledger/lrukv.go b/ledger/lrukv.go index 45f4f5027d..30530fe6bd 100644 --- a/ledger/lrukv.go +++ b/ledger/lrukv.go @@ -17,12 +17,13 @@ package ledger import ( + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" ) //msgp:ignore cachedKVData type cachedKVData struct { - persistedKVData + store.PersistedKVData // kv key key string @@ -62,11 +63,11 @@ func (m *lruKV) init(log logging.Logger, pendingWrites int, pendingWritesWarnThr // read the persistedKVData object that the lruKV has for the given key. // thread locking semantics : read lock -func (m *lruKV) read(key string) (data persistedKVData, has bool) { +func (m *lruKV) read(key string) (data store.PersistedKVData, has bool) { if el := m.kvs[key]; el != nil { - return el.Value.persistedKVData, true + return el.Value.PersistedKVData, true } - return persistedKVData{}, false + return store.PersistedKVData{}, false } // flushPendingWrites flushes the pending writes to the main lruKV cache. @@ -79,7 +80,7 @@ func (m *lruKV) flushPendingWrites() { for ; pendingEntriesCount > 0; pendingEntriesCount-- { select { case pendingKVData := <-m.pendingKVs: - m.write(pendingKVData.persistedKVData, pendingKVData.key) + m.write(pendingKVData.PersistedKVData, pendingKVData.key) default: return } @@ -89,9 +90,9 @@ func (m *lruKV) flushPendingWrites() { // writePending write a single persistedKVData entry to the pendingKVs buffer. // the function doesn't block, and in case of a buffer overflow the entry would not be added. // thread locking semantics : no lock is required. -func (m *lruKV) writePending(kv persistedKVData, key string) { +func (m *lruKV) writePending(kv store.PersistedKVData, key string) { select { - case m.pendingKVs <- cachedKVData{persistedKVData: kv, key: key}: + case m.pendingKVs <- cachedKVData{PersistedKVData: kv, key: key}: default: } } @@ -101,17 +102,17 @@ func (m *lruKV) writePending(kv persistedKVData, key string) { // version of what's already on the cache or not. In all cases, the entry is going // to be promoted to the front of the list. // thread locking semantics : write lock -func (m *lruKV) write(kvData persistedKVData, key string) { +func (m *lruKV) write(kvData store.PersistedKVData, key string) { if el := m.kvs[key]; el != nil { // already exists; is it a newer ? - if el.Value.before(&kvData) { + if el.Value.Before(&kvData) { // we update with a newer version. - el.Value = &cachedKVData{persistedKVData: kvData, key: key} + el.Value = &cachedKVData{PersistedKVData: kvData, key: key} } m.kvList.moveToFront(el) } else { // new entry. - m.kvs[key] = m.kvList.pushFront(&cachedKVData{persistedKVData: kvData, key: key}) + m.kvs[key] = m.kvList.pushFront(&cachedKVData{PersistedKVData: kvData, key: key}) } } diff --git a/ledger/lrukv_test.go b/ledger/lrukv_test.go index d266167310..01ed002065 100644 --- a/ledger/lrukv_test.go +++ b/ledger/lrukv_test.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -39,9 +40,9 @@ func TestLRUBasicKV(t *testing.T) { // write 50 KVs for i := 0; i < kvNum; i++ { kvValue := fmt.Sprintf("kv %d value", i) - kv := persistedKVData{ - value: []byte(kvValue), - round: basics.Round(i), + kv := store.PersistedKVData{ + Value: []byte(kvValue), + Round: basics.Round(i), } baseKV.write(kv, fmt.Sprintf("key%d", i)) } @@ -50,15 +51,15 @@ func TestLRUBasicKV(t *testing.T) { for i := 0; i < kvNum; i++ { kv, has := baseKV.read(fmt.Sprintf("key%d", i)) require.True(t, has) - require.Equal(t, basics.Round(i), kv.round) - require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.value)) + require.Equal(t, basics.Round(i), kv.Round) + require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.Value)) } // verify expected missing entries for i := kvNum; i < kvNum*2; i++ { kv, has := baseKV.read(fmt.Sprintf("key%d", i)) require.False(t, has) - require.Equal(t, persistedKVData{}, kv) + require.Equal(t, store.PersistedKVData{}, kv) } baseKV.prune(kvNum / 2) @@ -70,11 +71,11 @@ func TestLRUBasicKV(t *testing.T) { if i >= kvNum/2 && i < kvNum { // expected to have it. require.True(t, has) - require.Equal(t, basics.Round(i), kv.round) - require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.value)) + require.Equal(t, basics.Round(i), kv.Round) + require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.Value)) } else { require.False(t, has) - require.Equal(t, persistedKVData{}, kv) + require.Equal(t, store.PersistedKVData{}, kv) } } } @@ -90,9 +91,9 @@ func TestLRUKVPendingWrites(t *testing.T) { go func(i int) { time.Sleep(time.Duration((crypto.RandUint64() % 50)) * time.Millisecond) kvValue := fmt.Sprintf("kv %d value", i) - kv := persistedKVData{ - value: []byte(kvValue), - round: basics.Round(i), + kv := store.PersistedKVData{ + Value: []byte(kvValue), + Round: basics.Round(i), } baseKV.writePending(kv, fmt.Sprintf("key%d", i)) }(i) @@ -141,9 +142,9 @@ func TestLRUKVPendingWritesWarning(t *testing.T) { for j := 0; j < 50; j++ { for i := 0; i < j; i++ { kvValue := fmt.Sprintf("kv %d value", i) - kv := persistedKVData{ - value: []byte(kvValue), - round: basics.Round(i), + kv := store.PersistedKVData{ + Value: []byte(kvValue), + Round: basics.Round(i), } baseKV.writePending(kv, fmt.Sprintf("key%d", i)) } @@ -166,9 +167,9 @@ func TestLRUKVOmittedPendingWrites(t *testing.T) { for i := 0; i < pendingWritesBuffer*2; i++ { kvValue := fmt.Sprintf("kv %d value", i) - kv := persistedKVData{ - value: []byte(kvValue), - round: basics.Round(i), + kv := store.PersistedKVData{ + Value: []byte(kvValue), + Round: basics.Round(i), } baseKV.writePending(kv, fmt.Sprintf("key%d", i)) } @@ -179,15 +180,15 @@ func TestLRUKVOmittedPendingWrites(t *testing.T) { for i := 0; i < pendingWritesBuffer; i++ { kv, has := baseKV.read(fmt.Sprintf("key%d", i)) require.True(t, has) - require.Equal(t, basics.Round(i), kv.round) - require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.value)) + require.Equal(t, basics.Round(i), kv.Round) + require.Equal(t, fmt.Sprintf("kv %d value", i), string(kv.Value)) } // verify expected missing entries for i := pendingWritesBuffer; i < pendingWritesBuffer*2; i++ { kv, has := baseKV.read(fmt.Sprintf("key%d", i)) require.False(t, has) - require.Equal(t, persistedKVData{}, kv) + require.Equal(t, store.PersistedKVData{}, kv) } } @@ -218,7 +219,7 @@ func benchLruWriteKVs(b *testing.B, fillerKVs []cachedKVData, kvs []cachedKVData func fillLRUKV(baseKV lruKV, fillerKVs []cachedKVData) lruKV { for _, entry := range fillerKVs { - baseKV.write(entry.persistedKVData, entry.key) + baseKV.write(entry.PersistedKVData, entry.key) } return baseKV } @@ -229,9 +230,9 @@ func generatePersistedKVData(startRound, endRound int) []cachedKVData { kvValue := fmt.Sprintf("kv %d value", i) kvs[i-startRound] = cachedKVData{ - persistedKVData: persistedKVData{ - value: []byte(kvValue), - round: basics.Round(i + startRound), + PersistedKVData: store.PersistedKVData{ + Value: []byte(kvValue), + Round: basics.Round(i + startRound), }, key: fmt.Sprintf("key%d", i), } diff --git a/ledger/lruonlineaccts.go b/ledger/lruonlineaccts.go index ae05c497c5..bc8ccedfd4 100644 --- a/ledger/lruonlineaccts.go +++ b/ledger/lruonlineaccts.go @@ -18,6 +18,7 @@ package ledger import ( "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" ) @@ -32,7 +33,7 @@ type lruOnlineAccounts struct { accounts map[basics.Address]*persistedOnlineAccountDataListNode // pendingAccounts are used as a way to avoid taking a write-lock. When the caller needs to "materialize" these, // it would call flushPendingWrites and these would be merged into the accounts/accountsList - pendingAccounts chan persistedOnlineAccountData + pendingAccounts chan store.PersistedOnlineAccountData // log interface; used for logging the threshold event. log logging.Logger // pendingWritesWarnThreshold is the threshold beyond we would write a warning for exceeding the number of pendingAccounts entries @@ -44,18 +45,18 @@ type lruOnlineAccounts struct { func (m *lruOnlineAccounts) init(log logging.Logger, pendingWrites int, pendingWritesWarnThreshold int) { m.accountsList = newPersistedOnlineAccountList().allocateFreeNodes(pendingWrites) m.accounts = make(map[basics.Address]*persistedOnlineAccountDataListNode, pendingWrites) - m.pendingAccounts = make(chan persistedOnlineAccountData, pendingWrites) + m.pendingAccounts = make(chan store.PersistedOnlineAccountData, pendingWrites) m.log = log m.pendingWritesWarnThreshold = pendingWritesWarnThreshold } // read the persistedAccountData object that the lruAccounts has for the given address. // thread locking semantics : read lock -func (m *lruOnlineAccounts) read(addr basics.Address) (data persistedOnlineAccountData, has bool) { +func (m *lruOnlineAccounts) read(addr basics.Address) (data store.PersistedOnlineAccountData, has bool) { if el := m.accounts[addr]; el != nil { return *el.Value, true } - return persistedOnlineAccountData{}, false + return store.PersistedOnlineAccountData{}, false } // flushPendingWrites flushes the pending writes to the main lruAccounts cache. @@ -78,7 +79,7 @@ func (m *lruOnlineAccounts) flushPendingWrites() { // writePending write a single persistedOnlineAccountData entry to the pendingAccounts buffer. // the function doesn't block, and in case of a buffer overflow the entry would not be added. // thread locking semantics : no lock is required. -func (m *lruOnlineAccounts) writePending(acct persistedOnlineAccountData) { +func (m *lruOnlineAccounts) writePending(acct store.PersistedOnlineAccountData) { select { case m.pendingAccounts <- acct: default: @@ -90,17 +91,17 @@ func (m *lruOnlineAccounts) writePending(acct persistedOnlineAccountData) { // version of what's already on the cache or not. In all cases, the entry is going // to be promoted to the front of the list. // thread locking semantics : write lock -func (m *lruOnlineAccounts) write(acctData persistedOnlineAccountData) { - if el := m.accounts[acctData.addr]; el != nil { +func (m *lruOnlineAccounts) write(acctData store.PersistedOnlineAccountData) { + if el := m.accounts[acctData.Addr]; el != nil { // already exists; is it a newer ? - if el.Value.before(&acctData) { + if el.Value.Before(&acctData) { // we update with a newer version. el.Value = &acctData } m.accountsList.moveToFront(el) } else { // new entry. - m.accounts[acctData.addr] = m.accountsList.pushFront(&acctData) + m.accounts[acctData.Addr] = m.accountsList.pushFront(&acctData) } } @@ -113,7 +114,7 @@ func (m *lruOnlineAccounts) prune(newSize int) (removed int) { break } back := m.accountsList.back() - delete(m.accounts, back.Value.addr) + delete(m.accounts, back.Value.Addr) m.accountsList.remove(back) removed++ } diff --git a/ledger/lruonlineaccts_test.go b/ledger/lruonlineaccts_test.go index ca846f6c59..2b9994480e 100644 --- a/ledger/lruonlineaccts_test.go +++ b/ledger/lruonlineaccts_test.go @@ -24,6 +24,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -37,11 +38,11 @@ func TestLRUOnlineAccountsBasic(t *testing.T) { accountsNum := 50 // write 50 accounts for i := 0; i < accountsNum; i++ { - acct := persistedOnlineAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedOnlineAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseOnlineAcct.write(acct) } @@ -51,10 +52,10 @@ func TestLRUOnlineAccountsBasic(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseOnlineAcct.read(addr) require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } // verify expected missing entries @@ -62,7 +63,7 @@ func TestLRUOnlineAccountsBasic(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseOnlineAcct.read(addr) require.False(t, has) - require.Equal(t, persistedOnlineAccountData{}, acct) + require.Equal(t, store.PersistedOnlineAccountData{}, acct) } baseOnlineAcct.prune(accountsNum / 2) @@ -75,13 +76,13 @@ func TestLRUOnlineAccountsBasic(t *testing.T) { if i >= accountsNum/2 && i < accountsNum { // expected to have it. require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } else { require.False(t, has) - require.Equal(t, persistedOnlineAccountData{}, acct) + require.Equal(t, store.PersistedOnlineAccountData{}, acct) } } } @@ -96,11 +97,11 @@ func TestLRUOnlineAccountsPendingWrites(t *testing.T) { for i := 0; i < accountsNum; i++ { go func(i int) { time.Sleep(time.Duration((crypto.RandUint64() % 50)) * time.Millisecond) - acct := persistedOnlineAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedOnlineAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseOnlineAcct.writePending(acct) }(i) @@ -138,11 +139,11 @@ func TestLRUOnlineAccountsPendingWritesWarning(t *testing.T) { baseOnlineAcct.init(log, pendingWritesBuffer, pendingWritesThreshold) for j := 0; j < 50; j++ { for i := 0; i < j; i++ { - acct := persistedOnlineAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedOnlineAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseOnlineAcct.writePending(acct) } @@ -164,11 +165,11 @@ func TestLRUOnlineAccountsOmittedPendingWrites(t *testing.T) { baseOnlineAcct.init(log, pendingWritesBuffer, pendingWritesThreshold) for i := 0; i < pendingWritesBuffer*2; i++ { - acct := persistedOnlineAccountData{ - addr: basics.Address(crypto.Hash([]byte{byte(i)})), - round: basics.Round(i), - rowid: int64(i), - accountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, + acct := store.PersistedOnlineAccountData{ + Addr: basics.Address(crypto.Hash([]byte{byte(i)})), + Round: basics.Round(i), + Rowid: int64(i), + AccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}}, } baseOnlineAcct.writePending(acct) } @@ -180,10 +181,10 @@ func TestLRUOnlineAccountsOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseOnlineAcct.read(addr) require.True(t, has) - require.Equal(t, basics.Round(i), acct.round) - require.Equal(t, addr, acct.addr) - require.Equal(t, uint64(i), acct.accountData.MicroAlgos.Raw) - require.Equal(t, int64(i), acct.rowid) + require.Equal(t, basics.Round(i), acct.Round) + require.Equal(t, addr, acct.Addr) + require.Equal(t, uint64(i), acct.AccountData.MicroAlgos.Raw) + require.Equal(t, int64(i), acct.Rowid) } // verify expected missing entries @@ -191,6 +192,6 @@ func TestLRUOnlineAccountsOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) acct, has := baseOnlineAcct.read(addr) require.False(t, has) - require.Equal(t, persistedOnlineAccountData{}, acct) + require.Equal(t, store.PersistedOnlineAccountData{}, acct) } } diff --git a/ledger/lruresources.go b/ledger/lruresources.go index 70a2a4c14e..b2f9a63b04 100644 --- a/ledger/lruresources.go +++ b/ledger/lruresources.go @@ -18,12 +18,13 @@ package ledger import ( "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" ) //msgp:ignore cachedResourceData type cachedResourceData struct { - persistedResourcesData + store.PersistedResourcesData address basics.Address } @@ -67,11 +68,11 @@ func (m *lruResources) init(log logging.Logger, pendingWrites int, pendingWrites // read the persistedResourcesData object that the lruResources has for the given address and creatable index. // thread locking semantics : read lock -func (m *lruResources) read(addr basics.Address, aidx basics.CreatableIndex) (data persistedResourcesData, has bool) { +func (m *lruResources) read(addr basics.Address, aidx basics.CreatableIndex) (data store.PersistedResourcesData, has bool) { if el := m.resources[accountCreatable{address: addr, index: aidx}]; el != nil { - return el.Value.persistedResourcesData, true + return el.Value.PersistedResourcesData, true } - return persistedResourcesData{}, false + return store.PersistedResourcesData{}, false } // readNotFound returns whether we have attempted to read this address but it did not exist in the db. @@ -83,10 +84,10 @@ func (m *lruResources) readNotFound(addr basics.Address, idx basics.CreatableInd // read the persistedResourcesData object that the lruResources has for the given address. // thread locking semantics : read lock -func (m *lruResources) readAll(addr basics.Address) (ret []persistedResourcesData) { +func (m *lruResources) readAll(addr basics.Address) (ret []store.PersistedResourcesData) { for ac, pd := range m.resources { if ac.address == addr { - ret = append(ret, pd.Value.persistedResourcesData) + ret = append(ret, pd.Value.PersistedResourcesData) } } return @@ -104,7 +105,7 @@ outer: for ; pendingEntriesCount > 0; pendingEntriesCount-- { select { case pendingResourceData := <-m.pendingResources: - m.write(pendingResourceData.persistedResourcesData, pendingResourceData.address) + m.write(pendingResourceData.PersistedResourcesData, pendingResourceData.address) default: break outer } @@ -125,9 +126,9 @@ outer2: // writePending write a single persistedAccountData entry to the pendingResources buffer. // the function doesn't block, and in case of a buffer overflow the entry would not be added. // thread locking semantics : no lock is required. -func (m *lruResources) writePending(acct persistedResourcesData, addr basics.Address) { +func (m *lruResources) writePending(acct store.PersistedResourcesData, addr basics.Address) { select { - case m.pendingResources <- cachedResourceData{persistedResourcesData: acct, address: addr}: + case m.pendingResources <- cachedResourceData{PersistedResourcesData: acct, address: addr}: default: } } @@ -147,17 +148,17 @@ func (m *lruResources) writeNotFoundPending(addr basics.Address, idx basics.Crea // version of what's already on the cache or not. In all cases, the entry is going // to be promoted to the front of the list. // thread locking semantics : write lock -func (m *lruResources) write(resData persistedResourcesData, addr basics.Address) { - if el := m.resources[accountCreatable{address: addr, index: resData.aidx}]; el != nil { +func (m *lruResources) write(resData store.PersistedResourcesData, addr basics.Address) { + if el := m.resources[accountCreatable{address: addr, index: resData.Aidx}]; el != nil { // already exists; is it a newer ? - if el.Value.before(&resData) { + if el.Value.Before(&resData) { // we update with a newer version. - el.Value = &cachedResourceData{persistedResourcesData: resData, address: addr} + el.Value = &cachedResourceData{PersistedResourcesData: resData, address: addr} } m.resourcesList.moveToFront(el) } else { // new entry. - m.resources[accountCreatable{address: addr, index: resData.aidx}] = m.resourcesList.pushFront(&cachedResourceData{persistedResourcesData: resData, address: addr}) + m.resources[accountCreatable{address: addr, index: resData.Aidx}] = m.resourcesList.pushFront(&cachedResourceData{PersistedResourcesData: resData, address: addr}) } } @@ -170,7 +171,7 @@ func (m *lruResources) prune(newSize int) (removed int) { break } back := m.resourcesList.back() - delete(m.resources, accountCreatable{address: back.Value.address, index: back.Value.aidx}) + delete(m.resources, accountCreatable{address: back.Value.address, index: back.Value.Aidx}) m.resourcesList.remove(back) removed++ } diff --git a/ledger/lruresources_test.go b/ledger/lruresources_test.go index a389bc516c..695d1f026b 100644 --- a/ledger/lruresources_test.go +++ b/ledger/lruresources_test.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -39,11 +40,11 @@ func TestLRUBasicResources(t *testing.T) { // write 50 resources for i := 0; i < resourcesNum; i++ { addr := basics.Address(crypto.Hash([]byte{byte(i)})) - res := persistedResourcesData{ - addrid: int64(i), - aidx: basics.CreatableIndex(i), - round: basics.Round(i), - data: resourcesData{Total: uint64(i)}, + res := store.PersistedResourcesData{ + Addrid: int64(i), + Aidx: basics.CreatableIndex(i), + Round: basics.Round(i), + Data: store.ResourcesData{Total: uint64(i)}, } baseRes.write(res, addr) } @@ -53,10 +54,10 @@ func TestLRUBasicResources(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) res, has := baseRes.read(addr, basics.CreatableIndex(i)) require.True(t, has) - require.Equal(t, basics.Round(i), res.round) - require.Equal(t, int64(i), res.addrid) - require.Equal(t, uint64(i), res.data.Total) - require.Equal(t, basics.CreatableIndex(i), res.aidx) + require.Equal(t, basics.Round(i), res.Round) + require.Equal(t, int64(i), res.Addrid) + require.Equal(t, uint64(i), res.Data.Total) + require.Equal(t, basics.CreatableIndex(i), res.Aidx) } // verify expected missing entries @@ -64,7 +65,7 @@ func TestLRUBasicResources(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) res, has := baseRes.read(addr, basics.CreatableIndex(i%resourcesNum)) require.False(t, has) - require.Equal(t, persistedResourcesData{}, res) + require.Equal(t, store.PersistedResourcesData{}, res) } baseRes.prune(resourcesNum / 2) @@ -77,13 +78,13 @@ func TestLRUBasicResources(t *testing.T) { if i >= resourcesNum/2 && i < resourcesNum { // expected to have it. require.True(t, has) - require.Equal(t, basics.Round(i), res.round) - require.Equal(t, int64(i), res.addrid) - require.Equal(t, uint64(i), res.data.Total) - require.Equal(t, basics.CreatableIndex(i), res.aidx) + require.Equal(t, basics.Round(i), res.Round) + require.Equal(t, int64(i), res.Addrid) + require.Equal(t, uint64(i), res.Data.Total) + require.Equal(t, basics.CreatableIndex(i), res.Aidx) } else { require.False(t, has) - require.Equal(t, persistedResourcesData{}, res) + require.Equal(t, store.PersistedResourcesData{}, res) } } } @@ -99,11 +100,11 @@ func TestLRUResourcesPendingWrites(t *testing.T) { go func(i int) { time.Sleep(time.Duration((crypto.RandUint64() % 50)) * time.Millisecond) addr := basics.Address(crypto.Hash([]byte{byte(i)})) - res := persistedResourcesData{ - addrid: int64(i), - aidx: basics.CreatableIndex(i), - round: basics.Round(i), - data: resourcesData{Total: uint64(i)}, + res := store.PersistedResourcesData{ + Addrid: int64(i), + Aidx: basics.CreatableIndex(i), + Round: basics.Round(i), + Data: store.ResourcesData{Total: uint64(i)}, } baseRes.writePending(res, addr) }(i) @@ -152,11 +153,11 @@ func TestLRUResourcesPendingWritesWarning(t *testing.T) { for j := 0; j < 50; j++ { for i := 0; i < j; i++ { addr := basics.Address(crypto.Hash([]byte{byte(i)})) - res := persistedResourcesData{ - addrid: int64(i), - aidx: basics.CreatableIndex(i), - round: basics.Round(i), - data: resourcesData{Total: uint64(i)}, + res := store.PersistedResourcesData{ + Addrid: int64(i), + Aidx: basics.CreatableIndex(i), + Round: basics.Round(i), + Data: store.ResourcesData{Total: uint64(i)}, } baseRes.writePending(res, addr) } @@ -179,11 +180,11 @@ func TestLRUResourcesOmittedPendingWrites(t *testing.T) { for i := 0; i < pendingWritesBuffer*2; i++ { addr := basics.Address(crypto.Hash([]byte{byte(i)})) - res := persistedResourcesData{ - addrid: int64(i), - aidx: basics.CreatableIndex(i), - round: basics.Round(i), - data: resourcesData{Total: uint64(i)}, + res := store.PersistedResourcesData{ + Addrid: int64(i), + Aidx: basics.CreatableIndex(i), + Round: basics.Round(i), + Data: store.ResourcesData{Total: uint64(i)}, } baseRes.writePending(res, addr) } @@ -195,10 +196,10 @@ func TestLRUResourcesOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) res, has := baseRes.read(addr, basics.CreatableIndex(i)) require.True(t, has) - require.Equal(t, basics.Round(i), res.round) - require.Equal(t, int64(i), res.addrid) - require.Equal(t, uint64(i), res.data.Total) - require.Equal(t, basics.CreatableIndex(i), res.aidx) + require.Equal(t, basics.Round(i), res.Round) + require.Equal(t, int64(i), res.Addrid) + require.Equal(t, uint64(i), res.Data.Total) + require.Equal(t, basics.CreatableIndex(i), res.Aidx) } // verify expected missing entries @@ -206,7 +207,7 @@ func TestLRUResourcesOmittedPendingWrites(t *testing.T) { addr := basics.Address(crypto.Hash([]byte{byte(i)})) res, has := baseRes.read(addr, basics.CreatableIndex(i)) require.False(t, has) - require.Equal(t, persistedResourcesData{}, res) + require.Equal(t, store.PersistedResourcesData{}, res) } } @@ -237,7 +238,7 @@ func benchLruWriteResources(b *testing.B, fillerAccounts []cachedResourceData, a func fillLRUResources(baseRes lruResources, fillerAccounts []cachedResourceData) lruResources { for _, entry := range fillerAccounts { - baseRes.write(entry.persistedResourcesData, entry.address) + baseRes.write(entry.PersistedResourcesData, entry.address) } return baseRes } @@ -251,11 +252,11 @@ func generatePersistedResourcesData(startRound, endRound int) []cachedResourceDa digest := crypto.Hash(buffer) accounts[i-startRound] = cachedResourceData{ - persistedResourcesData: persistedResourcesData{ - addrid: int64(i), - aidx: basics.CreatableIndex(i), - round: basics.Round(i + startRound), - data: resourcesData{Total: uint64(i)}, + PersistedResourcesData: store.PersistedResourcesData{ + Addrid: int64(i), + Aidx: basics.CreatableIndex(i), + Round: basics.Round(i + startRound), + Data: store.ResourcesData{Total: uint64(i)}, }, address: basics.Address(digest), } diff --git a/ledger/msgp_gen.go b/ledger/msgp_gen.go index 9a1be6eb6c..d9e9cdbc57 100644 --- a/ledger/msgp_gen.go +++ b/ledger/msgp_gen.go @@ -7,7 +7,6 @@ import ( "github.com/algorand/msgp/msgp" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" ) @@ -29,30 +28,6 @@ import ( // |-----> (*) Msgsize // |-----> (*) MsgIsZero // -// baseAccountData -// |-----> (*) MarshalMsg -// |-----> (*) CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> (*) Msgsize -// |-----> (*) MsgIsZero -// -// baseOnlineAccountData -// |-----> (*) MarshalMsg -// |-----> (*) CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> (*) Msgsize -// |-----> (*) MsgIsZero -// -// baseVotingData -// |-----> (*) MarshalMsg -// |-----> (*) CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> (*) Msgsize -// |-----> (*) MsgIsZero -// // catchpointFileBalancesChunkV5 // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -117,22 +92,6 @@ import ( // |-----> Msgsize // |-----> MsgIsZero // -// resourceFlags -// |-----> MarshalMsg -// |-----> CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> Msgsize -// |-----> MsgIsZero -// -// resourcesData -// |-----> (*) MarshalMsg -// |-----> (*) CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> (*) Msgsize -// |-----> (*) MsgIsZero -// // txTailRound // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -487,2165 +446,169 @@ func (z *CatchpointFileHeader) MsgIsZero() bool { } // MarshalMsg implements msgp.Marshaler -func (z *baseAccountData) MarshalMsg(b []byte) (o []byte) { +func (z *catchpointFileBalancesChunkV5) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(21) - var zb0001Mask uint32 /* 23 bits */ - if (*z).baseVotingData.VoteID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x1 - } - if (*z).baseVotingData.SelectionID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).baseVotingData.VoteFirstValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x4 - } - if (*z).baseVotingData.VoteLastValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x8 - } - if (*z).baseVotingData.VoteKeyDilution == 0 { - zb0001Len-- - zb0001Mask |= 0x10 - } - if (*z).baseVotingData.StateProofID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x20 - } - if (*z).Status.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x100 - } - if (*z).MicroAlgos.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x200 - } - if (*z).RewardsBase == 0 { - zb0001Len-- - zb0001Mask |= 0x400 - } - if (*z).RewardedMicroAlgos.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x800 - } - if (*z).AuthAddr.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x1000 - } - if (*z).TotalAppSchemaNumUint == 0 { - zb0001Len-- - zb0001Mask |= 0x2000 - } - if (*z).TotalAppSchemaNumByteSlice == 0 { - zb0001Len-- - zb0001Mask |= 0x4000 - } - if (*z).TotalExtraAppPages == 0 { - zb0001Len-- - zb0001Mask |= 0x8000 - } - if (*z).TotalAssetParams == 0 { - zb0001Len-- - zb0001Mask |= 0x10000 - } - if (*z).TotalAssets == 0 { - zb0001Len-- - zb0001Mask |= 0x20000 - } - if (*z).TotalAppParams == 0 { - zb0001Len-- - zb0001Mask |= 0x40000 - } - if (*z).TotalAppLocalStates == 0 { - zb0001Len-- - zb0001Mask |= 0x80000 - } - if (*z).TotalBoxes == 0 { - zb0001Len-- - zb0001Mask |= 0x100000 - } - if (*z).TotalBoxBytes == 0 { - zb0001Len-- - zb0001Mask |= 0x200000 - } - if (*z).UpdateRound == 0 { - zb0001Len-- - zb0001Mask |= 0x400000 + zb0002Len := uint32(1) + var zb0002Mask uint8 /* 2 bits */ + if len((*z).Balances) == 0 { + zb0002Len-- + zb0002Mask |= 0x2 } - // variable map header, size zb0001Len - o = msgp.AppendMapHeader(o, zb0001Len) - if zb0001Len != 0 { - if (zb0001Mask & 0x1) == 0 { // if not empty - // string "A" - o = append(o, 0xa1, 0x41) - o = (*z).baseVotingData.VoteID.MarshalMsg(o) - } - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "B" - o = append(o, 0xa1, 0x42) - o = (*z).baseVotingData.SelectionID.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "C" - o = append(o, 0xa1, 0x43) - o = (*z).baseVotingData.VoteFirstValid.MarshalMsg(o) - } - if (zb0001Mask & 0x8) == 0 { // if not empty - // string "D" - o = append(o, 0xa1, 0x44) - o = (*z).baseVotingData.VoteLastValid.MarshalMsg(o) - } - if (zb0001Mask & 0x10) == 0 { // if not empty - // string "E" - o = append(o, 0xa1, 0x45) - o = msgp.AppendUint64(o, (*z).baseVotingData.VoteKeyDilution) - } - if (zb0001Mask & 0x20) == 0 { // if not empty - // string "F" - o = append(o, 0xa1, 0x46) - o = (*z).baseVotingData.StateProofID.MarshalMsg(o) - } - if (zb0001Mask & 0x100) == 0 { // if not empty - // string "a" - o = append(o, 0xa1, 0x61) - o = (*z).Status.MarshalMsg(o) - } - if (zb0001Mask & 0x200) == 0 { // if not empty - // string "b" - o = append(o, 0xa1, 0x62) - o = (*z).MicroAlgos.MarshalMsg(o) - } - if (zb0001Mask & 0x400) == 0 { // if not empty - // string "c" - o = append(o, 0xa1, 0x63) - o = msgp.AppendUint64(o, (*z).RewardsBase) - } - if (zb0001Mask & 0x800) == 0 { // if not empty - // string "d" - o = append(o, 0xa1, 0x64) - o = (*z).RewardedMicroAlgos.MarshalMsg(o) - } - if (zb0001Mask & 0x1000) == 0 { // if not empty - // string "e" - o = append(o, 0xa1, 0x65) - o = (*z).AuthAddr.MarshalMsg(o) - } - if (zb0001Mask & 0x2000) == 0 { // if not empty - // string "f" - o = append(o, 0xa1, 0x66) - o = msgp.AppendUint64(o, (*z).TotalAppSchemaNumUint) - } - if (zb0001Mask & 0x4000) == 0 { // if not empty - // string "g" - o = append(o, 0xa1, 0x67) - o = msgp.AppendUint64(o, (*z).TotalAppSchemaNumByteSlice) - } - if (zb0001Mask & 0x8000) == 0 { // if not empty - // string "h" - o = append(o, 0xa1, 0x68) - o = msgp.AppendUint32(o, (*z).TotalExtraAppPages) - } - if (zb0001Mask & 0x10000) == 0 { // if not empty - // string "i" - o = append(o, 0xa1, 0x69) - o = msgp.AppendUint64(o, (*z).TotalAssetParams) - } - if (zb0001Mask & 0x20000) == 0 { // if not empty - // string "j" - o = append(o, 0xa1, 0x6a) - o = msgp.AppendUint64(o, (*z).TotalAssets) - } - if (zb0001Mask & 0x40000) == 0 { // if not empty - // string "k" - o = append(o, 0xa1, 0x6b) - o = msgp.AppendUint64(o, (*z).TotalAppParams) - } - if (zb0001Mask & 0x80000) == 0 { // if not empty - // string "l" - o = append(o, 0xa1, 0x6c) - o = msgp.AppendUint64(o, (*z).TotalAppLocalStates) - } - if (zb0001Mask & 0x100000) == 0 { // if not empty - // string "m" - o = append(o, 0xa1, 0x6d) - o = msgp.AppendUint64(o, (*z).TotalBoxes) - } - if (zb0001Mask & 0x200000) == 0 { // if not empty - // string "n" - o = append(o, 0xa1, 0x6e) - o = msgp.AppendUint64(o, (*z).TotalBoxBytes) - } - if (zb0001Mask & 0x400000) == 0 { // if not empty - // string "z" - o = append(o, 0xa1, 0x7a) - o = msgp.AppendUint64(o, (*z).UpdateRound) + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if zb0002Len != 0 { + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "bl" + o = append(o, 0xa2, 0x62, 0x6c) + if (*z).Balances == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).Balances))) + } + for zb0001 := range (*z).Balances { + // omitempty: check for empty values + zb0003Len := uint32(2) + var zb0003Mask uint8 /* 3 bits */ + if (*z).Balances[zb0001].AccountData.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x2 + } + if (*z).Balances[zb0001].Address.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x4 + } + // variable map header, size zb0003Len + o = append(o, 0x80|uint8(zb0003Len)) + if (zb0003Mask & 0x2) == 0 { // if not empty + // string "ad" + o = append(o, 0xa2, 0x61, 0x64) + o = (*z).Balances[zb0001].AccountData.MarshalMsg(o) + } + if (zb0003Mask & 0x4) == 0 { // if not empty + // string "pk" + o = append(o, 0xa2, 0x70, 0x6b) + o = (*z).Balances[zb0001].Address.MarshalMsg(o) + } + } } } return } -func (_ *baseAccountData) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*baseAccountData) +func (_ *catchpointFileBalancesChunkV5) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFileBalancesChunkV5) return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *baseAccountData) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *catchpointFileBalancesChunkV5) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).Status.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Status") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "MicroAlgos") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "RewardsBase") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).RewardedMicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "RewardedMicroAlgos") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).AuthAddr.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "AuthAddr") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAppSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAppSchemaNumUint") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAppSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAppSchemaNumByteSlice") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalExtraAppPages, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalExtraAppPages") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAssetParams, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAssetParams") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAssets, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAssets") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAppParams, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAppParams") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAppLocalStates, bts, err = msgp.ReadUint64Bytes(bts) + if zb0002 > 0 { + zb0002-- + var zb0004 int + var zb0005 bool + zb0004, zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAppLocalStates") + err = msgp.WrapError(err, "struct-from-array", "Balances") return } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalBoxes, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalBoxes") + if zb0004 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0004), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "struct-from-array", "Balances") return } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalBoxBytes, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalBoxBytes") - return + if zb0005 { + (*z).Balances = nil + } else if (*z).Balances != nil && cap((*z).Balances) >= zb0004 { + (*z).Balances = ((*z).Balances)[:zb0004] + } else { + (*z).Balances = make([]encodedBalanceRecordV5, zb0004) } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SelectionID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).baseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "StateProofID") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "UpdateRound") - return - } - } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 { - (*z) = baseAccountData{} - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "a": - bts, err = (*z).Status.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Status") - return - } - case "b": - bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "MicroAlgos") - return - } - case "c": - (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "RewardsBase") - return - } - case "d": - bts, err = (*z).RewardedMicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "RewardedMicroAlgos") - return - } - case "e": - bts, err = (*z).AuthAddr.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "AuthAddr") - return - } - case "f": - (*z).TotalAppSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAppSchemaNumUint") - return - } - case "g": - (*z).TotalAppSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAppSchemaNumByteSlice") - return - } - case "h": - (*z).TotalExtraAppPages, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalExtraAppPages") - return - } - case "i": - (*z).TotalAssetParams, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAssetParams") - return - } - case "j": - (*z).TotalAssets, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAssets") - return - } - case "k": - (*z).TotalAppParams, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAppParams") - return - } - case "l": - (*z).TotalAppLocalStates, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAppLocalStates") - return - } - case "m": - (*z).TotalBoxes, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalBoxes") - return - } - case "n": - (*z).TotalBoxBytes, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalBoxBytes") - return - } - case "A": - bts, err = (*z).baseVotingData.VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteID") - return - } - case "B": - bts, err = (*z).baseVotingData.SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "SelectionID") - return - } - case "C": - bts, err = (*z).baseVotingData.VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteFirstValid") - return - } - case "D": - bts, err = (*z).baseVotingData.VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteLastValid") - return - } - case "E": - (*z).baseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "VoteKeyDilution") - return - } - case "F": - bts, err = (*z).baseVotingData.StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "StateProofID") - return - } - case "z": - (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "UpdateRound") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *baseAccountData) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*baseAccountData) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *baseAccountData) Msgsize() (s int) { - s = 3 + 2 + (*z).Status.Msgsize() + 2 + (*z).MicroAlgos.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).RewardedMicroAlgos.Msgsize() + 2 + (*z).AuthAddr.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + (*z).baseVotingData.VoteID.Msgsize() + 2 + (*z).baseVotingData.SelectionID.Msgsize() + 2 + (*z).baseVotingData.VoteFirstValid.Msgsize() + 2 + (*z).baseVotingData.VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).baseVotingData.StateProofID.Msgsize() + 2 + msgp.Uint64Size - return -} - -// MsgIsZero returns whether this is a zero value -func (z *baseAccountData) MsgIsZero() bool { - return ((*z).Status.MsgIsZero()) && ((*z).MicroAlgos.MsgIsZero()) && ((*z).RewardsBase == 0) && ((*z).RewardedMicroAlgos.MsgIsZero()) && ((*z).AuthAddr.MsgIsZero()) && ((*z).TotalAppSchemaNumUint == 0) && ((*z).TotalAppSchemaNumByteSlice == 0) && ((*z).TotalExtraAppPages == 0) && ((*z).TotalAssetParams == 0) && ((*z).TotalAssets == 0) && ((*z).TotalAppParams == 0) && ((*z).TotalAppLocalStates == 0) && ((*z).TotalBoxes == 0) && ((*z).TotalBoxBytes == 0) && ((*z).baseVotingData.VoteID.MsgIsZero()) && ((*z).baseVotingData.SelectionID.MsgIsZero()) && ((*z).baseVotingData.VoteFirstValid.MsgIsZero()) && ((*z).baseVotingData.VoteLastValid.MsgIsZero()) && ((*z).baseVotingData.VoteKeyDilution == 0) && ((*z).baseVotingData.StateProofID.MsgIsZero()) && ((*z).UpdateRound == 0) -} - -// MarshalMsg implements msgp.Marshaler -func (z *baseOnlineAccountData) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(8) - var zb0001Mask uint16 /* 10 bits */ - if (*z).baseVotingData.VoteID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x1 - } - if (*z).baseVotingData.SelectionID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).baseVotingData.VoteFirstValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x4 - } - if (*z).baseVotingData.VoteLastValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x8 - } - if (*z).baseVotingData.VoteKeyDilution == 0 { - zb0001Len-- - zb0001Mask |= 0x10 - } - if (*z).baseVotingData.StateProofID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x20 - } - if (*z).MicroAlgos.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x40 - } - if (*z).RewardsBase == 0 { - zb0001Len-- - zb0001Mask |= 0x80 - } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x1) == 0 { // if not empty - // string "A" - o = append(o, 0xa1, 0x41) - o = (*z).baseVotingData.VoteID.MarshalMsg(o) - } - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "B" - o = append(o, 0xa1, 0x42) - o = (*z).baseVotingData.SelectionID.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "C" - o = append(o, 0xa1, 0x43) - o = (*z).baseVotingData.VoteFirstValid.MarshalMsg(o) - } - if (zb0001Mask & 0x8) == 0 { // if not empty - // string "D" - o = append(o, 0xa1, 0x44) - o = (*z).baseVotingData.VoteLastValid.MarshalMsg(o) - } - if (zb0001Mask & 0x10) == 0 { // if not empty - // string "E" - o = append(o, 0xa1, 0x45) - o = msgp.AppendUint64(o, (*z).baseVotingData.VoteKeyDilution) - } - if (zb0001Mask & 0x20) == 0 { // if not empty - // string "F" - o = append(o, 0xa1, 0x46) - o = (*z).baseVotingData.StateProofID.MarshalMsg(o) - } - if (zb0001Mask & 0x40) == 0 { // if not empty - // string "Y" - o = append(o, 0xa1, 0x59) - o = (*z).MicroAlgos.MarshalMsg(o) - } - if (zb0001Mask & 0x80) == 0 { // if not empty - // string "Z" - o = append(o, 0xa1, 0x5a) - o = msgp.AppendUint64(o, (*z).RewardsBase) - } - } - return -} - -func (_ *baseOnlineAccountData) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*baseOnlineAccountData) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *baseOnlineAccountData) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SelectionID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).baseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).baseVotingData.StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "StateProofID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "MicroAlgos") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "RewardsBase") - return - } - } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 { - (*z) = baseOnlineAccountData{} - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "A": - bts, err = (*z).baseVotingData.VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteID") - return - } - case "B": - bts, err = (*z).baseVotingData.SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "SelectionID") - return - } - case "C": - bts, err = (*z).baseVotingData.VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteFirstValid") - return - } - case "D": - bts, err = (*z).baseVotingData.VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteLastValid") - return - } - case "E": - (*z).baseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "VoteKeyDilution") - return - } - case "F": - bts, err = (*z).baseVotingData.StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "StateProofID") - return - } - case "Y": - bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "MicroAlgos") - return - } - case "Z": - (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "RewardsBase") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *baseOnlineAccountData) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*baseOnlineAccountData) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *baseOnlineAccountData) Msgsize() (s int) { - s = 1 + 2 + (*z).baseVotingData.VoteID.Msgsize() + 2 + (*z).baseVotingData.SelectionID.Msgsize() + 2 + (*z).baseVotingData.VoteFirstValid.Msgsize() + 2 + (*z).baseVotingData.VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).baseVotingData.StateProofID.Msgsize() + 2 + (*z).MicroAlgos.Msgsize() + 2 + msgp.Uint64Size - return -} - -// MsgIsZero returns whether this is a zero value -func (z *baseOnlineAccountData) MsgIsZero() bool { - return ((*z).baseVotingData.VoteID.MsgIsZero()) && ((*z).baseVotingData.SelectionID.MsgIsZero()) && ((*z).baseVotingData.VoteFirstValid.MsgIsZero()) && ((*z).baseVotingData.VoteLastValid.MsgIsZero()) && ((*z).baseVotingData.VoteKeyDilution == 0) && ((*z).baseVotingData.StateProofID.MsgIsZero()) && ((*z).MicroAlgos.MsgIsZero()) && ((*z).RewardsBase == 0) -} - -// MarshalMsg implements msgp.Marshaler -func (z *baseVotingData) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(6) - var zb0001Mask uint8 /* 7 bits */ - if (*z).VoteID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x1 - } - if (*z).SelectionID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).VoteFirstValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x4 - } - if (*z).VoteLastValid.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x8 - } - if (*z).VoteKeyDilution == 0 { - zb0001Len-- - zb0001Mask |= 0x10 - } - if (*z).StateProofID.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x20 - } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x1) == 0 { // if not empty - // string "A" - o = append(o, 0xa1, 0x41) - o = (*z).VoteID.MarshalMsg(o) - } - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "B" - o = append(o, 0xa1, 0x42) - o = (*z).SelectionID.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "C" - o = append(o, 0xa1, 0x43) - o = (*z).VoteFirstValid.MarshalMsg(o) - } - if (zb0001Mask & 0x8) == 0 { // if not empty - // string "D" - o = append(o, 0xa1, 0x44) - o = (*z).VoteLastValid.MarshalMsg(o) - } - if (zb0001Mask & 0x10) == 0 { // if not empty - // string "E" - o = append(o, 0xa1, 0x45) - o = msgp.AppendUint64(o, (*z).VoteKeyDilution) - } - if (zb0001Mask & 0x20) == 0 { // if not empty - // string "F" - o = append(o, 0xa1, 0x46) - o = (*z).StateProofID.MarshalMsg(o) - } - } - return -} - -func (_ *baseVotingData) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*baseVotingData) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *baseVotingData) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SelectionID") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "StateProofID") - return - } - } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 { - (*z) = baseVotingData{} - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "A": - bts, err = (*z).VoteID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteID") - return - } - case "B": - bts, err = (*z).SelectionID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "SelectionID") - return - } - case "C": - bts, err = (*z).VoteFirstValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteFirstValid") - return - } - case "D": - bts, err = (*z).VoteLastValid.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "VoteLastValid") - return - } - case "E": - (*z).VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "VoteKeyDilution") - return - } - case "F": - bts, err = (*z).StateProofID.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "StateProofID") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *baseVotingData) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*baseVotingData) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *baseVotingData) Msgsize() (s int) { - s = 1 + 2 + (*z).VoteID.Msgsize() + 2 + (*z).SelectionID.Msgsize() + 2 + (*z).VoteFirstValid.Msgsize() + 2 + (*z).VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).StateProofID.Msgsize() - return -} - -// MsgIsZero returns whether this is a zero value -func (z *baseVotingData) MsgIsZero() bool { - return ((*z).VoteID.MsgIsZero()) && ((*z).SelectionID.MsgIsZero()) && ((*z).VoteFirstValid.MsgIsZero()) && ((*z).VoteLastValid.MsgIsZero()) && ((*z).VoteKeyDilution == 0) && ((*z).StateProofID.MsgIsZero()) -} - -// MarshalMsg implements msgp.Marshaler -func (z *catchpointFileBalancesChunkV5) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0002Len := uint32(1) - var zb0002Mask uint8 /* 2 bits */ - if len((*z).Balances) == 0 { - zb0002Len-- - zb0002Mask |= 0x2 - } - // variable map header, size zb0002Len - o = append(o, 0x80|uint8(zb0002Len)) - if zb0002Len != 0 { - if (zb0002Mask & 0x2) == 0 { // if not empty - // string "bl" - o = append(o, 0xa2, 0x62, 0x6c) - if (*z).Balances == nil { - o = msgp.AppendNil(o) - } else { - o = msgp.AppendArrayHeader(o, uint32(len((*z).Balances))) - } - for zb0001 := range (*z).Balances { - // omitempty: check for empty values - zb0003Len := uint32(2) - var zb0003Mask uint8 /* 3 bits */ - if (*z).Balances[zb0001].AccountData.MsgIsZero() { - zb0003Len-- - zb0003Mask |= 0x2 - } - if (*z).Balances[zb0001].Address.MsgIsZero() { - zb0003Len-- - zb0003Mask |= 0x4 - } - // variable map header, size zb0003Len - o = append(o, 0x80|uint8(zb0003Len)) - if (zb0003Mask & 0x2) == 0 { // if not empty - // string "ad" - o = append(o, 0xa2, 0x61, 0x64) - o = (*z).Balances[zb0001].AccountData.MarshalMsg(o) - } - if (zb0003Mask & 0x4) == 0 { // if not empty - // string "pk" - o = append(o, 0xa2, 0x70, 0x6b) - o = (*z).Balances[zb0001].Address.MarshalMsg(o) - } - } - } - } - return -} - -func (_ *catchpointFileBalancesChunkV5) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFileBalancesChunkV5) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *catchpointFileBalancesChunkV5) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0002 int - var zb0003 bool - zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 > 0 { - zb0002-- - var zb0004 int - var zb0005 bool - zb0004, zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances") - return - } - if zb0004 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0004), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "struct-from-array", "Balances") - return - } - if zb0005 { - (*z).Balances = nil - } else if (*z).Balances != nil && cap((*z).Balances) >= zb0004 { - (*z).Balances = ((*z).Balances)[:zb0004] - } else { - (*z).Balances = make([]encodedBalanceRecordV5, zb0004) - } - for zb0001 := range (*z).Balances { - var zb0006 int - var zb0007 bool - zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) - return - } - if zb0006 > 0 { - zb0006-- - bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array", "Address") - return - } - } - if zb0006 > 0 { - zb0006-- - bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array", "AccountData") - return - } - } - if zb0006 > 0 { - err = msgp.ErrTooManyArrayFields(zb0006) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) - return - } - if zb0007 { - (*z).Balances[zb0001] = encodedBalanceRecordV5{} - } - for zb0006 > 0 { - zb0006-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) - return - } - switch string(field) { - case "pk": - bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "Address") - return - } - case "ad": - bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "AccountData") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) - return - } - } - } - } - } - } - if zb0002 > 0 { - err = msgp.ErrTooManyArrayFields(zb0002) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0003 { - (*z) = catchpointFileBalancesChunkV5{} - } - for zb0002 > 0 { - zb0002-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "bl": - var zb0008 int - var zb0009 bool - zb0008, zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Balances") - return - } - if zb0008 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0008), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "Balances") - return - } - if zb0009 { - (*z).Balances = nil - } else if (*z).Balances != nil && cap((*z).Balances) >= zb0008 { - (*z).Balances = ((*z).Balances)[:zb0008] - } else { - (*z).Balances = make([]encodedBalanceRecordV5, zb0008) - } - for zb0001 := range (*z).Balances { - var zb0010 int - var zb0011 bool - zb0010, zb0011, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0010, zb0011, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001) - return - } - if zb0010 > 0 { - zb0010-- - bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array", "Address") - return - } - } - if zb0010 > 0 { - zb0010-- - bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array", "AccountData") - return - } - } - if zb0010 > 0 { - err = msgp.ErrTooManyArrayFields(zb0010) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001) - return - } - if zb0011 { - (*z).Balances[zb0001] = encodedBalanceRecordV5{} - } - for zb0010 > 0 { - zb0010-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001) - return - } - switch string(field) { - case "pk": - bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001, "Address") - return - } - case "ad": - bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001, "AccountData") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001) - return - } - } - } - } - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *catchpointFileBalancesChunkV5) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFileBalancesChunkV5) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *catchpointFileBalancesChunkV5) Msgsize() (s int) { - s = 1 + 3 + msgp.ArrayHeaderSize - for zb0001 := range (*z).Balances { - s += 1 + 3 + (*z).Balances[zb0001].Address.Msgsize() + 3 + (*z).Balances[zb0001].AccountData.Msgsize() - } - return -} - -// MsgIsZero returns whether this is a zero value -func (z *catchpointFileBalancesChunkV5) MsgIsZero() bool { - return (len((*z).Balances) == 0) -} - -// MarshalMsg implements msgp.Marshaler -func (z *catchpointFileChunkV6) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0003Len := uint32(2) - var zb0003Mask uint8 /* 4 bits */ - if len((*z).Balances) == 0 { - zb0003Len-- - zb0003Mask |= 0x2 - } - if len((*z).KVs) == 0 { - zb0003Len-- - zb0003Mask |= 0x4 - } - // variable map header, size zb0003Len - o = append(o, 0x80|uint8(zb0003Len)) - if zb0003Len != 0 { - if (zb0003Mask & 0x2) == 0 { // if not empty - // string "bl" - o = append(o, 0xa2, 0x62, 0x6c) - if (*z).Balances == nil { - o = msgp.AppendNil(o) - } else { - o = msgp.AppendArrayHeader(o, uint32(len((*z).Balances))) - } - for zb0001 := range (*z).Balances { - o = (*z).Balances[zb0001].MarshalMsg(o) - } - } - if (zb0003Mask & 0x4) == 0 { // if not empty - // string "kv" - o = append(o, 0xa2, 0x6b, 0x76) - if (*z).KVs == nil { - o = msgp.AppendNil(o) - } else { - o = msgp.AppendArrayHeader(o, uint32(len((*z).KVs))) - } - for zb0002 := range (*z).KVs { - // omitempty: check for empty values - zb0004Len := uint32(2) - var zb0004Mask uint8 /* 3 bits */ - if len((*z).KVs[zb0002].Key) == 0 { - zb0004Len-- - zb0004Mask |= 0x2 - } - if len((*z).KVs[zb0002].Value) == 0 { - zb0004Len-- - zb0004Mask |= 0x4 - } - // variable map header, size zb0004Len - o = append(o, 0x80|uint8(zb0004Len)) - if (zb0004Mask & 0x2) == 0 { // if not empty - // string "k" - o = append(o, 0xa1, 0x6b) - o = msgp.AppendBytes(o, (*z).KVs[zb0002].Key) - } - if (zb0004Mask & 0x4) == 0 { // if not empty - // string "v" - o = append(o, 0xa1, 0x76) - o = msgp.AppendBytes(o, (*z).KVs[zb0002].Value) - } - } - } - } - return -} - -func (_ *catchpointFileChunkV6) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFileChunkV6) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *catchpointFileChunkV6) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0003 int - var zb0004 bool - zb0003, zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0003, zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0003 > 0 { - zb0003-- - var zb0005 int - var zb0006 bool - zb0005, zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances") - return - } - if zb0005 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0005), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "struct-from-array", "Balances") - return - } - if zb0006 { - (*z).Balances = nil - } else if (*z).Balances != nil && cap((*z).Balances) >= zb0005 { - (*z).Balances = ((*z).Balances)[:zb0005] - } else { - (*z).Balances = make([]encodedBalanceRecordV6, zb0005) - } - for zb0001 := range (*z).Balances { - bts, err = (*z).Balances[zb0001].UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) - return - } - } - } - if zb0003 > 0 { - zb0003-- - var zb0007 int - var zb0008 bool - zb0007, zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs") - return - } - if zb0007 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0007), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "struct-from-array", "KVs") - return - } - if zb0008 { - (*z).KVs = nil - } else if (*z).KVs != nil && cap((*z).KVs) >= zb0007 { - (*z).KVs = ((*z).KVs)[:zb0007] - } else { - (*z).KVs = make([]encodedKVRecordV6, zb0007) - } - for zb0002 := range (*z).KVs { - var zb0009 int - var zb0010 bool - zb0009, zb0010, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0009, zb0010, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) - return - } - if zb0009 > 0 { - zb0009-- - var zb0011 int - zb0011, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Key") - return - } - if zb0011 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0011), uint64(encodedKVRecordV6MaxKeyLength)) - return - } - (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Key") - return - } - } - if zb0009 > 0 { - zb0009-- - var zb0012 int - zb0012, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Value") - return - } - if zb0012 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0012), uint64(encodedKVRecordV6MaxValueLength)) - return - } - (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Value") - return - } - } - if zb0009 > 0 { - err = msgp.ErrTooManyArrayFields(zb0009) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) - return - } - if zb0010 { - (*z).KVs[zb0002] = encodedKVRecordV6{} - } - for zb0009 > 0 { - zb0009-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) - return - } - switch string(field) { - case "k": - var zb0013 int - zb0013, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Key") - return - } - if zb0013 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0013), uint64(encodedKVRecordV6MaxKeyLength)) - return - } - (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Key") - return - } - case "v": - var zb0014 int - zb0014, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Value") - return - } - if zb0014 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0014), uint64(encodedKVRecordV6MaxValueLength)) - return - } - (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Value") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) - return - } - } - } - } - } - } - if zb0003 > 0 { - err = msgp.ErrTooManyArrayFields(zb0003) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0004 { - (*z) = catchpointFileChunkV6{} - } - for zb0003 > 0 { - zb0003-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "bl": - var zb0015 int - var zb0016 bool - zb0015, zb0016, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Balances") - return - } - if zb0015 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0015), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "Balances") - return - } - if zb0016 { - (*z).Balances = nil - } else if (*z).Balances != nil && cap((*z).Balances) >= zb0015 { - (*z).Balances = ((*z).Balances)[:zb0015] - } else { - (*z).Balances = make([]encodedBalanceRecordV6, zb0015) - } - for zb0001 := range (*z).Balances { - bts, err = (*z).Balances[zb0001].UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Balances", zb0001) - return - } - } - case "kv": - var zb0017 int - var zb0018 bool - zb0017, zb0018, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "KVs") - return - } - if zb0017 > BalancesPerCatchpointFileChunk { - err = msgp.ErrOverflow(uint64(zb0017), uint64(BalancesPerCatchpointFileChunk)) - err = msgp.WrapError(err, "KVs") - return - } - if zb0018 { - (*z).KVs = nil - } else if (*z).KVs != nil && cap((*z).KVs) >= zb0017 { - (*z).KVs = ((*z).KVs)[:zb0017] - } else { - (*z).KVs = make([]encodedKVRecordV6, zb0017) - } - for zb0002 := range (*z).KVs { - var zb0019 int - var zb0020 bool - zb0019, zb0020, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0019, zb0020, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002) - return - } - if zb0019 > 0 { - zb0019-- - var zb0021 int - zb0021, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Key") - return - } - if zb0021 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0021), uint64(encodedKVRecordV6MaxKeyLength)) - return - } - (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Key") - return - } - } - if zb0019 > 0 { - zb0019-- - var zb0022 int - zb0022, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Value") - return - } - if zb0022 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0022), uint64(encodedKVRecordV6MaxValueLength)) - return - } - (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Value") - return - } - } - if zb0019 > 0 { - err = msgp.ErrTooManyArrayFields(zb0019) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002) - return - } - if zb0020 { - (*z).KVs[zb0002] = encodedKVRecordV6{} - } - for zb0019 > 0 { - zb0019-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002) - return - } - switch string(field) { - case "k": - var zb0023 int - zb0023, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "Key") - return - } - if zb0023 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0023), uint64(encodedKVRecordV6MaxKeyLength)) - return - } - (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "Key") - return - } - case "v": - var zb0024 int - zb0024, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "Value") - return - } - if zb0024 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0024), uint64(encodedKVRecordV6MaxValueLength)) - return - } - (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002, "Value") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err, "KVs", zb0002) - return - } - } - } - } - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *catchpointFileChunkV6) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFileChunkV6) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *catchpointFileChunkV6) Msgsize() (s int) { - s = 1 + 3 + msgp.ArrayHeaderSize - for zb0001 := range (*z).Balances { - s += (*z).Balances[zb0001].Msgsize() - } - s += 3 + msgp.ArrayHeaderSize - for zb0002 := range (*z).KVs { - s += 1 + 2 + msgp.BytesPrefixSize + len((*z).KVs[zb0002].Key) + 2 + msgp.BytesPrefixSize + len((*z).KVs[zb0002].Value) - } - return -} - -// MsgIsZero returns whether this is a zero value -func (z *catchpointFileChunkV6) MsgIsZero() bool { - return (len((*z).Balances) == 0) && (len((*z).KVs) == 0) -} - -// MarshalMsg implements msgp.Marshaler -func (z *catchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(6) - var zb0001Mask uint8 /* 7 bits */ - if (*z).Totals.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).TotalAccounts == 0 { - zb0001Len-- - zb0001Mask |= 0x4 - } - if (*z).BiggestChunkLen == 0 { - zb0001Len-- - zb0001Mask |= 0x8 - } - if (*z).TotalChunks == 0 { - zb0001Len-- - zb0001Mask |= 0x10 - } - if (*z).TotalKVs == 0 { - zb0001Len-- - zb0001Mask |= 0x20 - } - if (*z).TrieBalancesHash.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x40 - } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "accountTotals" - o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) - o = (*z).Totals.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "accountsCount" - o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalAccounts) - } - if (zb0001Mask & 0x8) == 0 { // if not empty - // string "biggestChunk" - o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b) - o = msgp.AppendUint64(o, (*z).BiggestChunkLen) - } - if (zb0001Mask & 0x10) == 0 { // if not empty - // string "chunksCount" - o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalChunks) - } - if (zb0001Mask & 0x20) == 0 { // if not empty - // string "kvsCount" - o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalKVs) - } - if (zb0001Mask & 0x40) == 0 { // if not empty - // string "trieBalancesHash" - o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68) - o = (*z).TrieBalancesHash.MarshalMsg(o) - } - } - return -} - -func (_ *catchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFirstStageInfo) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *catchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).Totals.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Totals") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAccounts") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalKVs") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalChunks") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen") - return - } - } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 { - (*z) = catchpointFirstStageInfo{} - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "accountTotals": - bts, err = (*z).Totals.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Totals") - return - } - case "trieBalancesHash": - bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "TrieBalancesHash") - return - } - case "accountsCount": - (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAccounts") - return - } - case "kvsCount": - (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalKVs") - return - } - case "chunksCount": - (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalChunks") - return - } - case "biggestChunk": - (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "BiggestChunkLen") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *catchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointFirstStageInfo) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *catchpointFirstStageInfo) Msgsize() (s int) { - s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size - return -} - -// MsgIsZero returns whether this is a zero value -func (z *catchpointFirstStageInfo) MsgIsZero() bool { - return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) -} - -// MarshalMsg implements msgp.Marshaler -func (z catchpointState) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendString(o, string(z)) - return -} - -func (_ catchpointState) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(catchpointState) - if !ok { - _, ok = (z).(*catchpointState) - } - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *catchpointState) UnmarshalMsg(bts []byte) (o []byte, err error) { - { - var zb0001 string - zb0001, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - (*z) = catchpointState(zb0001) - } - o = bts - return -} - -func (_ *catchpointState) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*catchpointState) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z catchpointState) Msgsize() (s int) { - s = msgp.StringPrefixSize + len(string(z)) - return -} - -// MsgIsZero returns whether this is a zero value -func (z catchpointState) MsgIsZero() bool { - return z == "" -} - -// MarshalMsg implements msgp.Marshaler -func (z *encodedBalanceRecordV5) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(2) - var zb0001Mask uint8 /* 3 bits */ - if (*z).AccountData.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).Address.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x4 - } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "ad" - o = append(o, 0xa2, 0x61, 0x64) - o = (*z).AccountData.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "pk" - o = append(o, 0xa2, 0x70, 0x6b) - o = (*z).Address.MarshalMsg(o) - } - } - return -} - -func (_ *encodedBalanceRecordV5) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedBalanceRecordV5) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *encodedBalanceRecordV5) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Address") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "AccountData") - return + for zb0001 := range (*z).Balances { + var zb0006 int + var zb0007 bool + zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) + return + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array", "Address") + return + } + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array", "AccountData") + return + } + } + if zb0006 > 0 { + err = msgp.ErrTooManyArrayFields(zb0006) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) + return + } + if zb0007 { + (*z).Balances[zb0001] = encodedBalanceRecordV5{} + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) + return + } + switch string(field) { + case "pk": + bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "Address") + return + } + case "ad": + bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001, "AccountData") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) + return + } + } + } + } } } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) if err != nil { err = msgp.WrapError(err, "struct-from-array") return @@ -2656,29 +619,108 @@ func (z *encodedBalanceRecordV5) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.WrapError(err) return } - if zb0002 { - (*z) = encodedBalanceRecordV5{} + if zb0003 { + (*z) = catchpointFileBalancesChunkV5{} } - for zb0001 > 0 { - zb0001-- + for zb0002 > 0 { + zb0002-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err) return } switch string(field) { - case "pk": - bts, err = (*z).Address.UnmarshalMsg(bts) + case "bl": + var zb0008 int + var zb0009 bool + zb0008, zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "Address") + err = msgp.WrapError(err, "Balances") return } - case "ad": - bts, err = (*z).AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "AccountData") + if zb0008 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0008), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "Balances") return } + if zb0009 { + (*z).Balances = nil + } else if (*z).Balances != nil && cap((*z).Balances) >= zb0008 { + (*z).Balances = ((*z).Balances)[:zb0008] + } else { + (*z).Balances = make([]encodedBalanceRecordV5, zb0008) + } + for zb0001 := range (*z).Balances { + var zb0010 int + var zb0011 bool + zb0010, zb0011, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0010, zb0011, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001) + return + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array", "Address") + return + } + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array", "AccountData") + return + } + } + if zb0010 > 0 { + err = msgp.ErrTooManyArrayFields(zb0010) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001) + return + } + if zb0011 { + (*z).Balances[zb0001] = encodedBalanceRecordV5{} + } + for zb0010 > 0 { + zb0010-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001) + return + } + switch string(field) { + case "pk": + bts, err = (*z).Balances[zb0001].Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001, "Address") + return + } + case "ad": + bts, err = (*z).Balances[zb0001].AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001, "AccountData") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001) + return + } + } + } + } + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -2692,93 +734,99 @@ func (z *encodedBalanceRecordV5) UnmarshalMsg(bts []byte) (o []byte, err error) return } -func (_ *encodedBalanceRecordV5) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedBalanceRecordV5) +func (_ *catchpointFileBalancesChunkV5) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFileBalancesChunkV5) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *encodedBalanceRecordV5) Msgsize() (s int) { - s = 1 + 3 + (*z).Address.Msgsize() + 3 + (*z).AccountData.Msgsize() +func (z *catchpointFileBalancesChunkV5) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + for zb0001 := range (*z).Balances { + s += 1 + 3 + (*z).Balances[zb0001].Address.Msgsize() + 3 + (*z).Balances[zb0001].AccountData.Msgsize() + } return } // MsgIsZero returns whether this is a zero value -func (z *encodedBalanceRecordV5) MsgIsZero() bool { - return ((*z).Address.MsgIsZero()) && ((*z).AccountData.MsgIsZero()) +func (z *catchpointFileBalancesChunkV5) MsgIsZero() bool { + return (len((*z).Balances) == 0) } // MarshalMsg implements msgp.Marshaler -func (z *encodedBalanceRecordV6) MarshalMsg(b []byte) (o []byte) { +func (z *catchpointFileChunkV6) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0003Len := uint32(4) - var zb0003Mask uint8 /* 5 bits */ - if (*z).Address.MsgIsZero() { + zb0003Len := uint32(2) + var zb0003Mask uint8 /* 4 bits */ + if len((*z).Balances) == 0 { zb0003Len-- zb0003Mask |= 0x2 } - if (*z).AccountData.MsgIsZero() { + if len((*z).KVs) == 0 { zb0003Len-- zb0003Mask |= 0x4 } - if len((*z).Resources) == 0 { - zb0003Len-- - zb0003Mask |= 0x8 - } - if (*z).ExpectingMoreEntries == false { - zb0003Len-- - zb0003Mask |= 0x10 - } // variable map header, size zb0003Len o = append(o, 0x80|uint8(zb0003Len)) if zb0003Len != 0 { if (zb0003Mask & 0x2) == 0 { // if not empty - // string "a" - o = append(o, 0xa1, 0x61) - o = (*z).Address.MarshalMsg(o) + // string "bl" + o = append(o, 0xa2, 0x62, 0x6c) + if (*z).Balances == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).Balances))) + } + for zb0001 := range (*z).Balances { + o = (*z).Balances[zb0001].MarshalMsg(o) + } } if (zb0003Mask & 0x4) == 0 { // if not empty - // string "b" - o = append(o, 0xa1, 0x62) - o = (*z).AccountData.MarshalMsg(o) - } - if (zb0003Mask & 0x8) == 0 { // if not empty - // string "c" - o = append(o, 0xa1, 0x63) - if (*z).Resources == nil { + // string "kv" + o = append(o, 0xa2, 0x6b, 0x76) + if (*z).KVs == nil { o = msgp.AppendNil(o) } else { - o = msgp.AppendMapHeader(o, uint32(len((*z).Resources))) - } - zb0001_keys := make([]uint64, 0, len((*z).Resources)) - for zb0001 := range (*z).Resources { - zb0001_keys = append(zb0001_keys, zb0001) + o = msgp.AppendArrayHeader(o, uint32(len((*z).KVs))) } - sort.Sort(SortUint64(zb0001_keys)) - for _, zb0001 := range zb0001_keys { - zb0002 := (*z).Resources[zb0001] - _ = zb0002 - o = msgp.AppendUint64(o, zb0001) - o = zb0002.MarshalMsg(o) + for zb0002 := range (*z).KVs { + // omitempty: check for empty values + zb0004Len := uint32(2) + var zb0004Mask uint8 /* 3 bits */ + if len((*z).KVs[zb0002].Key) == 0 { + zb0004Len-- + zb0004Mask |= 0x2 + } + if len((*z).KVs[zb0002].Value) == 0 { + zb0004Len-- + zb0004Mask |= 0x4 + } + // variable map header, size zb0004Len + o = append(o, 0x80|uint8(zb0004Len)) + if (zb0004Mask & 0x2) == 0 { // if not empty + // string "k" + o = append(o, 0xa1, 0x6b) + o = msgp.AppendBytes(o, (*z).KVs[zb0002].Key) + } + if (zb0004Mask & 0x4) == 0 { // if not empty + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendBytes(o, (*z).KVs[zb0002].Value) + } } } - if (zb0003Mask & 0x10) == 0 { // if not empty - // string "e" - o = append(o, 0xa1, 0x65) - o = msgp.AppendBool(o, (*z).ExpectingMoreEntries) - } } return } -func (_ *encodedBalanceRecordV6) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedBalanceRecordV6) +func (_ *catchpointFileChunkV6) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFileChunkV6) return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *catchpointFileChunkV6) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field var zb0003 int @@ -2790,65 +838,167 @@ func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.WrapError(err) return } - if zb0003 > 0 { - zb0003-- - bts, err = (*z).Address.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Address") - return - } - } - if zb0003 > 0 { - zb0003-- - bts, err = (*z).AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "AccountData") - return - } - } if zb0003 > 0 { zb0003-- var zb0005 int var zb0006 bool - zb0005, zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + zb0005, zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Resources") + err = msgp.WrapError(err, "struct-from-array", "Balances") return } - if zb0005 > resourcesPerCatchpointFileChunkBackwardCompatible { - err = msgp.ErrOverflow(uint64(zb0005), uint64(resourcesPerCatchpointFileChunkBackwardCompatible)) - err = msgp.WrapError(err, "struct-from-array", "Resources") + if zb0005 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0005), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "struct-from-array", "Balances") return } if zb0006 { - (*z).Resources = nil - } else if (*z).Resources == nil { - (*z).Resources = make(map[uint64]msgp.Raw, zb0005) + (*z).Balances = nil + } else if (*z).Balances != nil && cap((*z).Balances) >= zb0005 { + (*z).Balances = ((*z).Balances)[:zb0005] + } else { + (*z).Balances = make([]encodedBalanceRecordV6, zb0005) } - for zb0005 > 0 { - var zb0001 uint64 - var zb0002 msgp.Raw - zb0005-- - zb0001, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Resources") - return - } - bts, err = zb0002.UnmarshalMsg(bts) + for zb0001 := range (*z).Balances { + bts, err = (*z).Balances[zb0001].UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Resources", zb0001) + err = msgp.WrapError(err, "struct-from-array", "Balances", zb0001) return } - (*z).Resources[zb0001] = zb0002 } } if zb0003 > 0 { zb0003-- - (*z).ExpectingMoreEntries, bts, err = msgp.ReadBoolBytes(bts) + var zb0007 int + var zb0008 bool + zb0007, zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ExpectingMoreEntries") + err = msgp.WrapError(err, "struct-from-array", "KVs") + return + } + if zb0007 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0007), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "struct-from-array", "KVs") return } + if zb0008 { + (*z).KVs = nil + } else if (*z).KVs != nil && cap((*z).KVs) >= zb0007 { + (*z).KVs = ((*z).KVs)[:zb0007] + } else { + (*z).KVs = make([]encodedKVRecordV6, zb0007) + } + for zb0002 := range (*z).KVs { + var zb0009 int + var zb0010 bool + zb0009, zb0010, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0009, zb0010, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) + return + } + if zb0009 > 0 { + zb0009-- + var zb0011 int + zb0011, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Key") + return + } + if zb0011 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0011), uint64(encodedKVRecordV6MaxKeyLength)) + return + } + (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Key") + return + } + } + if zb0009 > 0 { + zb0009-- + var zb0012 int + zb0012, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Value") + return + } + if zb0012 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0012), uint64(encodedKVRecordV6MaxValueLength)) + return + } + (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array", "Value") + return + } + } + if zb0009 > 0 { + err = msgp.ErrTooManyArrayFields(zb0009) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) + return + } + if zb0010 { + (*z).KVs[zb0002] = encodedKVRecordV6{} + } + for zb0009 > 0 { + zb0009-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) + return + } + switch string(field) { + case "k": + var zb0013 int + zb0013, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Key") + return + } + if zb0013 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0013), uint64(encodedKVRecordV6MaxKeyLength)) + return + } + (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Key") + return + } + case "v": + var zb0014 int + zb0014, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Value") + return + } + if zb0014 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0014), uint64(encodedKVRecordV6MaxValueLength)) + return + } + (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002, "Value") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KVs", zb0002) + return + } + } + } + } + } } if zb0003 > 0 { err = msgp.ErrTooManyArrayFields(zb0003) @@ -2863,7 +1013,7 @@ func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) return } if zb0004 { - (*z) = encodedBalanceRecordV6{} + (*z) = catchpointFileChunkV6{} } for zb0003 > 0 { zb0003-- @@ -2873,57 +1023,163 @@ func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) return } switch string(field) { - case "a": - bts, err = (*z).Address.UnmarshalMsg(bts) + case "bl": + var zb0015 int + var zb0016 bool + zb0015, zb0016, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "Address") + err = msgp.WrapError(err, "Balances") return } - case "b": - bts, err = (*z).AccountData.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "AccountData") + if zb0015 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0015), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "Balances") return } - case "c": - var zb0007 int - var zb0008 bool - zb0007, zb0008, bts, err = msgp.ReadMapHeaderBytes(bts) + if zb0016 { + (*z).Balances = nil + } else if (*z).Balances != nil && cap((*z).Balances) >= zb0015 { + (*z).Balances = ((*z).Balances)[:zb0015] + } else { + (*z).Balances = make([]encodedBalanceRecordV6, zb0015) + } + for zb0001 := range (*z).Balances { + bts, err = (*z).Balances[zb0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Balances", zb0001) + return + } + } + case "kv": + var zb0017 int + var zb0018 bool + zb0017, zb0018, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "Resources") + err = msgp.WrapError(err, "KVs") return } - if zb0007 > resourcesPerCatchpointFileChunkBackwardCompatible { - err = msgp.ErrOverflow(uint64(zb0007), uint64(resourcesPerCatchpointFileChunkBackwardCompatible)) - err = msgp.WrapError(err, "Resources") + if zb0017 > BalancesPerCatchpointFileChunk { + err = msgp.ErrOverflow(uint64(zb0017), uint64(BalancesPerCatchpointFileChunk)) + err = msgp.WrapError(err, "KVs") return } - if zb0008 { - (*z).Resources = nil - } else if (*z).Resources == nil { - (*z).Resources = make(map[uint64]msgp.Raw, zb0007) + if zb0018 { + (*z).KVs = nil + } else if (*z).KVs != nil && cap((*z).KVs) >= zb0017 { + (*z).KVs = ((*z).KVs)[:zb0017] + } else { + (*z).KVs = make([]encodedKVRecordV6, zb0017) } - for zb0007 > 0 { - var zb0001 uint64 - var zb0002 msgp.Raw - zb0007-- - zb0001, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Resources") - return - } - bts, err = zb0002.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Resources", zb0001) - return + for zb0002 := range (*z).KVs { + var zb0019 int + var zb0020 bool + zb0019, zb0020, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0019, zb0020, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002) + return + } + if zb0019 > 0 { + zb0019-- + var zb0021 int + zb0021, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Key") + return + } + if zb0021 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0021), uint64(encodedKVRecordV6MaxKeyLength)) + return + } + (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Key") + return + } + } + if zb0019 > 0 { + zb0019-- + var zb0022 int + zb0022, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Value") + return + } + if zb0022 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0022), uint64(encodedKVRecordV6MaxValueLength)) + return + } + (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array", "Value") + return + } + } + if zb0019 > 0 { + err = msgp.ErrTooManyArrayFields(zb0019) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002) + return + } + if zb0020 { + (*z).KVs[zb0002] = encodedKVRecordV6{} + } + for zb0019 > 0 { + zb0019-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002) + return + } + switch string(field) { + case "k": + var zb0023 int + zb0023, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "Key") + return + } + if zb0023 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0023), uint64(encodedKVRecordV6MaxKeyLength)) + return + } + (*z).KVs[zb0002].Key, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Key) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "Key") + return + } + case "v": + var zb0024 int + zb0024, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "Value") + return + } + if zb0024 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0024), uint64(encodedKVRecordV6MaxValueLength)) + return + } + (*z).KVs[zb0002].Value, bts, err = msgp.ReadBytesBytes(bts, (*z).KVs[zb0002].Value) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002, "Value") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "KVs", zb0002) + return + } + } + } } - (*z).Resources[zb0001] = zb0002 - } - case "e": - (*z).ExpectingMoreEntries, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "ExpectingMoreEntries") - return } default: err = msgp.ErrNoField(string(field)) @@ -2938,68 +1194,103 @@ func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) return } -func (_ *encodedBalanceRecordV6) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedBalanceRecordV6) +func (_ *catchpointFileChunkV6) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFileChunkV6) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *encodedBalanceRecordV6) Msgsize() (s int) { - s = 1 + 2 + (*z).Address.Msgsize() + 2 + (*z).AccountData.Msgsize() + 2 + msgp.MapHeaderSize - if (*z).Resources != nil { - for zb0001, zb0002 := range (*z).Resources { - _ = zb0001 - _ = zb0002 - s += 0 + msgp.Uint64Size + zb0002.Msgsize() - } +func (z *catchpointFileChunkV6) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + for zb0001 := range (*z).Balances { + s += (*z).Balances[zb0001].Msgsize() + } + s += 3 + msgp.ArrayHeaderSize + for zb0002 := range (*z).KVs { + s += 1 + 2 + msgp.BytesPrefixSize + len((*z).KVs[zb0002].Key) + 2 + msgp.BytesPrefixSize + len((*z).KVs[zb0002].Value) } - s += 2 + msgp.BoolSize return } // MsgIsZero returns whether this is a zero value -func (z *encodedBalanceRecordV6) MsgIsZero() bool { - return ((*z).Address.MsgIsZero()) && ((*z).AccountData.MsgIsZero()) && (len((*z).Resources) == 0) && ((*z).ExpectingMoreEntries == false) +func (z *catchpointFileChunkV6) MsgIsZero() bool { + return (len((*z).Balances) == 0) && (len((*z).KVs) == 0) } // MarshalMsg implements msgp.Marshaler -func (z *encodedKVRecordV6) MarshalMsg(b []byte) (o []byte) { +func (z *catchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(2) - var zb0001Mask uint8 /* 3 bits */ - if len((*z).Key) == 0 { + zb0001Len := uint32(6) + var zb0001Mask uint8 /* 7 bits */ + if (*z).Totals.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2 } - if len((*z).Value) == 0 { + if (*z).TotalAccounts == 0 { zb0001Len-- zb0001Mask |= 0x4 } + if (*z).BiggestChunkLen == 0 { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).TotalChunks == 0 { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).TotalKVs == 0 { + zb0001Len-- + zb0001Mask |= 0x20 + } + if (*z).TrieBalancesHash.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40 + } // variable map header, size zb0001Len o = append(o, 0x80|uint8(zb0001Len)) if zb0001Len != 0 { if (zb0001Mask & 0x2) == 0 { // if not empty - // string "k" - o = append(o, 0xa1, 0x6b) - o = msgp.AppendBytes(o, (*z).Key) + // string "accountTotals" + o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + o = (*z).Totals.MarshalMsg(o) } if (zb0001Mask & 0x4) == 0 { // if not empty - // string "v" - o = append(o, 0xa1, 0x76) - o = msgp.AppendBytes(o, (*z).Value) + // string "accountsCount" + o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalAccounts) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "biggestChunk" + o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b) + o = msgp.AppendUint64(o, (*z).BiggestChunkLen) + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "chunksCount" + o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalChunks) + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "kvsCount" + o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalKVs) + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "trieBalancesHash" + o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68) + o = (*z).TrieBalancesHash.MarshalMsg(o) } } return } -func (_ *encodedKVRecordV6) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedKVRecordV6) +func (_ *catchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFirstStageInfo) return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *catchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field var zb0001 int @@ -3013,37 +1304,49 @@ func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - var zb0003 int - zb0003, err = msgp.ReadBytesBytesHeader(bts) + bts, err = (*z).Totals.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Key") + err = msgp.WrapError(err, "struct-from-array", "Totals") return } - if zb0003 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0003), uint64(encodedKVRecordV6MaxKeyLength)) + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash") return } - (*z).Key, bts, err = msgp.ReadBytesBytes(bts, (*z).Key) + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Key") + err = msgp.WrapError(err, "struct-from-array", "TotalAccounts") return } } if zb0001 > 0 { zb0001-- - var zb0004 int - zb0004, err = msgp.ReadBytesBytesHeader(bts) + (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Value") + err = msgp.WrapError(err, "struct-from-array", "TotalKVs") return } - if zb0004 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0004), uint64(encodedKVRecordV6MaxValueLength)) + } + if zb0001 > 0 { + zb0001-- + (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalChunks") return } - (*z).Value, bts, err = msgp.ReadBytesBytes(bts, (*z).Value) + } + if zb0001 > 0 { + zb0001-- + (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Value") + err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen") return } } @@ -3060,7 +1363,7 @@ func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { return } if zb0002 { - (*z) = encodedKVRecordV6{} + (*z) = catchpointFirstStageInfo{} } for zb0001 > 0 { zb0001-- @@ -3070,36 +1373,40 @@ func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { return } switch string(field) { - case "k": - var zb0005 int - zb0005, err = msgp.ReadBytesBytesHeader(bts) + case "accountTotals": + bts, err = (*z).Totals.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "Key") + err = msgp.WrapError(err, "Totals") return } - if zb0005 > encodedKVRecordV6MaxKeyLength { - err = msgp.ErrOverflow(uint64(zb0005), uint64(encodedKVRecordV6MaxKeyLength)) + case "trieBalancesHash": + bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TrieBalancesHash") return } - (*z).Key, bts, err = msgp.ReadBytesBytes(bts, (*z).Key) + case "accountsCount": + (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Key") + err = msgp.WrapError(err, "TotalAccounts") return } - case "v": - var zb0006 int - zb0006, err = msgp.ReadBytesBytesHeader(bts) + case "kvsCount": + (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Value") + err = msgp.WrapError(err, "TotalKVs") return } - if zb0006 > encodedKVRecordV6MaxValueLength { - err = msgp.ErrOverflow(uint64(zb0006), uint64(encodedKVRecordV6MaxValueLength)) + case "chunksCount": + (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalChunks") return } - (*z).Value, bts, err = msgp.ReadBytesBytes(bts, (*z).Value) + case "biggestChunk": + (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "Value") + err = msgp.WrapError(err, "BiggestChunkLen") return } default: @@ -3115,613 +1422,538 @@ func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -func (_ *encodedKVRecordV6) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*encodedKVRecordV6) +func (_ *catchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointFirstStageInfo) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *encodedKVRecordV6) Msgsize() (s int) { - s = 1 + 2 + msgp.BytesPrefixSize + len((*z).Key) + 2 + msgp.BytesPrefixSize + len((*z).Value) +func (z *catchpointFirstStageInfo) Msgsize() (s int) { + s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size return } // MsgIsZero returns whether this is a zero value -func (z *encodedKVRecordV6) MsgIsZero() bool { - return (len((*z).Key) == 0) && (len((*z).Value) == 0) +func (z *catchpointFirstStageInfo) MsgIsZero() bool { + return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) } // MarshalMsg implements msgp.Marshaler -func (z hashKind) MarshalMsg(b []byte) (o []byte) { +func (z catchpointState) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendByte(o, byte(z)) + o = msgp.AppendString(o, string(z)) return } -func (_ hashKind) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(hashKind) +func (_ catchpointState) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(catchpointState) if !ok { - _, ok = (z).(*hashKind) + _, ok = (z).(*catchpointState) } return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *hashKind) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *catchpointState) UnmarshalMsg(bts []byte) (o []byte, err error) { { - var zb0001 byte - zb0001, bts, err = msgp.ReadByteBytes(bts) + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) if err != nil { err = msgp.WrapError(err) return } - (*z) = hashKind(zb0001) + (*z) = catchpointState(zb0001) } o = bts return } -func (_ *hashKind) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*hashKind) +func (_ *catchpointState) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*catchpointState) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z hashKind) Msgsize() (s int) { - s = msgp.ByteSize +func (z catchpointState) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) return } // MsgIsZero returns whether this is a zero value -func (z hashKind) MsgIsZero() bool { - return z == 0 +func (z catchpointState) MsgIsZero() bool { + return z == "" } // MarshalMsg implements msgp.Marshaler -func (z resourceFlags) MarshalMsg(b []byte) (o []byte) { +func (z *encodedBalanceRecordV5) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendUint8(o, uint8(z)) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).AccountData.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Address.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "ad" + o = append(o, 0xa2, 0x61, 0x64) + o = (*z).AccountData.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "pk" + o = append(o, 0xa2, 0x70, 0x6b) + o = (*z).Address.MarshalMsg(o) + } + } return } -func (_ resourceFlags) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(resourceFlags) - if !ok { - _, ok = (z).(*resourceFlags) - } +func (_ *encodedBalanceRecordV5) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedBalanceRecordV5) return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *resourceFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { - { - var zb0001 uint8 - zb0001, bts, err = msgp.ReadUint8Bytes(bts) +func (z *encodedBalanceRecordV5) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Address") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "AccountData") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { if err != nil { err = msgp.WrapError(err) return } - (*z) = resourceFlags(zb0001) + if zb0002 { + (*z) = encodedBalanceRecordV5{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "pk": + bts, err = (*z).Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Address") + return + } + case "ad": + bts, err = (*z).AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "AccountData") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } } o = bts return } -func (_ *resourceFlags) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*resourceFlags) +func (_ *encodedBalanceRecordV5) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedBalanceRecordV5) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z resourceFlags) Msgsize() (s int) { - s = msgp.Uint8Size +func (z *encodedBalanceRecordV5) Msgsize() (s int) { + s = 1 + 3 + (*z).Address.Msgsize() + 3 + (*z).AccountData.Msgsize() return } // MsgIsZero returns whether this is a zero value -func (z resourceFlags) MsgIsZero() bool { - return z == 0 +func (z *encodedBalanceRecordV5) MsgIsZero() bool { + return ((*z).Address.MsgIsZero()) && ((*z).AccountData.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler -func (z *resourcesData) MarshalMsg(b []byte) (o []byte) { +func (z *encodedBalanceRecordV6) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0002Len := uint32(26) - var zb0002Mask uint32 /* 27 bits */ - if (*z).Total == 0 { - zb0002Len-- - zb0002Mask |= 0x2 - } - if (*z).Decimals == 0 { - zb0002Len-- - zb0002Mask |= 0x4 - } - if (*z).DefaultFrozen == false { - zb0002Len-- - zb0002Mask |= 0x8 - } - if (*z).UnitName == "" { - zb0002Len-- - zb0002Mask |= 0x10 - } - if (*z).AssetName == "" { - zb0002Len-- - zb0002Mask |= 0x20 - } - if (*z).URL == "" { - zb0002Len-- - zb0002Mask |= 0x40 - } - if (*z).MetadataHash == ([32]byte{}) { - zb0002Len-- - zb0002Mask |= 0x80 - } - if (*z).Manager.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x100 - } - if (*z).Reserve.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x200 - } - if (*z).Freeze.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x400 - } - if (*z).Clawback.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x800 - } - if (*z).Amount == 0 { - zb0002Len-- - zb0002Mask |= 0x1000 - } - if (*z).Frozen == false { - zb0002Len-- - zb0002Mask |= 0x2000 - } - if (*z).SchemaNumUint == 0 { - zb0002Len-- - zb0002Mask |= 0x4000 - } - if (*z).SchemaNumByteSlice == 0 { - zb0002Len-- - zb0002Mask |= 0x8000 - } - if (*z).KeyValue.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x10000 - } - if len((*z).ApprovalProgram) == 0 { - zb0002Len-- - zb0002Mask |= 0x20000 - } - if len((*z).ClearStateProgram) == 0 { - zb0002Len-- - zb0002Mask |= 0x40000 - } - if (*z).GlobalState.MsgIsZero() { - zb0002Len-- - zb0002Mask |= 0x80000 - } - if (*z).LocalStateSchemaNumUint == 0 { - zb0002Len-- - zb0002Mask |= 0x100000 - } - if (*z).LocalStateSchemaNumByteSlice == 0 { - zb0002Len-- - zb0002Mask |= 0x200000 - } - if (*z).GlobalStateSchemaNumUint == 0 { - zb0002Len-- - zb0002Mask |= 0x400000 - } - if (*z).GlobalStateSchemaNumByteSlice == 0 { - zb0002Len-- - zb0002Mask |= 0x800000 + zb0003Len := uint32(4) + var zb0003Mask uint8 /* 5 bits */ + if (*z).Address.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x2 } - if (*z).ExtraProgramPages == 0 { - zb0002Len-- - zb0002Mask |= 0x1000000 + if (*z).AccountData.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x4 } - if (*z).ResourceFlags == 0 { - zb0002Len-- - zb0002Mask |= 0x2000000 + if len((*z).Resources) == 0 { + zb0003Len-- + zb0003Mask |= 0x8 } - if (*z).UpdateRound == 0 { - zb0002Len-- - zb0002Mask |= 0x4000000 + if (*z).ExpectingMoreEntries == false { + zb0003Len-- + zb0003Mask |= 0x10 } - // variable map header, size zb0002Len - o = msgp.AppendMapHeader(o, zb0002Len) - if zb0002Len != 0 { - if (zb0002Mask & 0x2) == 0 { // if not empty + // variable map header, size zb0003Len + o = append(o, 0x80|uint8(zb0003Len)) + if zb0003Len != 0 { + if (zb0003Mask & 0x2) == 0 { // if not empty // string "a" o = append(o, 0xa1, 0x61) - o = msgp.AppendUint64(o, (*z).Total) + o = (*z).Address.MarshalMsg(o) } - if (zb0002Mask & 0x4) == 0 { // if not empty + if (zb0003Mask & 0x4) == 0 { // if not empty // string "b" o = append(o, 0xa1, 0x62) - o = msgp.AppendUint32(o, (*z).Decimals) + o = (*z).AccountData.MarshalMsg(o) } - if (zb0002Mask & 0x8) == 0 { // if not empty + if (zb0003Mask & 0x8) == 0 { // if not empty // string "c" o = append(o, 0xa1, 0x63) - o = msgp.AppendBool(o, (*z).DefaultFrozen) - } - if (zb0002Mask & 0x10) == 0 { // if not empty - // string "d" - o = append(o, 0xa1, 0x64) - o = msgp.AppendString(o, (*z).UnitName) - } - if (zb0002Mask & 0x20) == 0 { // if not empty - // string "e" - o = append(o, 0xa1, 0x65) - o = msgp.AppendString(o, (*z).AssetName) - } - if (zb0002Mask & 0x40) == 0 { // if not empty - // string "f" - o = append(o, 0xa1, 0x66) - o = msgp.AppendString(o, (*z).URL) - } - if (zb0002Mask & 0x80) == 0 { // if not empty - // string "g" - o = append(o, 0xa1, 0x67) - o = msgp.AppendBytes(o, ((*z).MetadataHash)[:]) - } - if (zb0002Mask & 0x100) == 0 { // if not empty - // string "h" - o = append(o, 0xa1, 0x68) - o = (*z).Manager.MarshalMsg(o) - } - if (zb0002Mask & 0x200) == 0 { // if not empty - // string "i" - o = append(o, 0xa1, 0x69) - o = (*z).Reserve.MarshalMsg(o) - } - if (zb0002Mask & 0x400) == 0 { // if not empty - // string "j" - o = append(o, 0xa1, 0x6a) - o = (*z).Freeze.MarshalMsg(o) - } - if (zb0002Mask & 0x800) == 0 { // if not empty - // string "k" - o = append(o, 0xa1, 0x6b) - o = (*z).Clawback.MarshalMsg(o) - } - if (zb0002Mask & 0x1000) == 0 { // if not empty - // string "l" - o = append(o, 0xa1, 0x6c) - o = msgp.AppendUint64(o, (*z).Amount) - } - if (zb0002Mask & 0x2000) == 0 { // if not empty - // string "m" - o = append(o, 0xa1, 0x6d) - o = msgp.AppendBool(o, (*z).Frozen) - } - if (zb0002Mask & 0x4000) == 0 { // if not empty - // string "n" - o = append(o, 0xa1, 0x6e) - o = msgp.AppendUint64(o, (*z).SchemaNumUint) - } - if (zb0002Mask & 0x8000) == 0 { // if not empty - // string "o" - o = append(o, 0xa1, 0x6f) - o = msgp.AppendUint64(o, (*z).SchemaNumByteSlice) - } - if (zb0002Mask & 0x10000) == 0 { // if not empty - // string "p" - o = append(o, 0xa1, 0x70) - o = (*z).KeyValue.MarshalMsg(o) - } - if (zb0002Mask & 0x20000) == 0 { // if not empty - // string "q" - o = append(o, 0xa1, 0x71) - o = msgp.AppendBytes(o, (*z).ApprovalProgram) - } - if (zb0002Mask & 0x40000) == 0 { // if not empty - // string "r" - o = append(o, 0xa1, 0x72) - o = msgp.AppendBytes(o, (*z).ClearStateProgram) - } - if (zb0002Mask & 0x80000) == 0 { // if not empty - // string "s" - o = append(o, 0xa1, 0x73) - o = (*z).GlobalState.MarshalMsg(o) - } - if (zb0002Mask & 0x100000) == 0 { // if not empty - // string "t" - o = append(o, 0xa1, 0x74) - o = msgp.AppendUint64(o, (*z).LocalStateSchemaNumUint) - } - if (zb0002Mask & 0x200000) == 0 { // if not empty - // string "u" - o = append(o, 0xa1, 0x75) - o = msgp.AppendUint64(o, (*z).LocalStateSchemaNumByteSlice) - } - if (zb0002Mask & 0x400000) == 0 { // if not empty - // string "v" - o = append(o, 0xa1, 0x76) - o = msgp.AppendUint64(o, (*z).GlobalStateSchemaNumUint) - } - if (zb0002Mask & 0x800000) == 0 { // if not empty - // string "w" - o = append(o, 0xa1, 0x77) - o = msgp.AppendUint64(o, (*z).GlobalStateSchemaNumByteSlice) - } - if (zb0002Mask & 0x1000000) == 0 { // if not empty - // string "x" - o = append(o, 0xa1, 0x78) - o = msgp.AppendUint32(o, (*z).ExtraProgramPages) - } - if (zb0002Mask & 0x2000000) == 0 { // if not empty - // string "y" - o = append(o, 0xa1, 0x79) - o = msgp.AppendUint8(o, uint8((*z).ResourceFlags)) + if (*z).Resources == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendMapHeader(o, uint32(len((*z).Resources))) + } + zb0001_keys := make([]uint64, 0, len((*z).Resources)) + for zb0001 := range (*z).Resources { + zb0001_keys = append(zb0001_keys, zb0001) + } + sort.Sort(SortUint64(zb0001_keys)) + for _, zb0001 := range zb0001_keys { + zb0002 := (*z).Resources[zb0001] + _ = zb0002 + o = msgp.AppendUint64(o, zb0001) + o = zb0002.MarshalMsg(o) + } } - if (zb0002Mask & 0x4000000) == 0 { // if not empty - // string "z" - o = append(o, 0xa1, 0x7a) - o = msgp.AppendUint64(o, (*z).UpdateRound) + if (zb0003Mask & 0x10) == 0 { // if not empty + // string "e" + o = append(o, 0xa1, 0x65) + o = msgp.AppendBool(o, (*z).ExpectingMoreEntries) } } return } -func (_ *resourcesData) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*resourcesData) +func (_ *encodedBalanceRecordV6) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedBalanceRecordV6) return ok } // UnmarshalMsg implements msgp.Unmarshaler -func (z *resourcesData) UnmarshalMsg(bts []byte) (o []byte, err error) { +func (z *encodedBalanceRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { var field []byte _ = field - var zb0002 int - var zb0003 bool - zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + var zb0003 int + var zb0004 bool + zb0003, zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) if _, ok := err.(msgp.TypeError); ok { - zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + zb0003, zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - if zb0002 > 0 { - zb0002-- - (*z).Total, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Total") - return - } - } - if zb0002 > 0 { - zb0002-- - (*z).Decimals, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Decimals") - return - } - } - if zb0002 > 0 { - zb0002-- - (*z).DefaultFrozen, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "DefaultFrozen") - return - } - } - if zb0002 > 0 { - zb0002-- - (*z).UnitName, bts, err = msgp.ReadStringBytes(bts) + if zb0003 > 0 { + zb0003-- + bts, err = (*z).Address.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "UnitName") + err = msgp.WrapError(err, "struct-from-array", "Address") return } } - if zb0002 > 0 { - zb0002-- - (*z).AssetName, bts, err = msgp.ReadStringBytes(bts) + if zb0003 > 0 { + zb0003-- + bts, err = (*z).AccountData.UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "AssetName") + err = msgp.WrapError(err, "struct-from-array", "AccountData") return } } - if zb0002 > 0 { - zb0002-- - (*z).URL, bts, err = msgp.ReadStringBytes(bts) + if zb0003 > 0 { + zb0003-- + var zb0005 int + var zb0006 bool + zb0005, zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "URL") + err = msgp.WrapError(err, "struct-from-array", "Resources") return } - } - if zb0002 > 0 { - zb0002-- - bts, err = msgp.ReadExactBytes(bts, ((*z).MetadataHash)[:]) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "MetadataHash") + if zb0005 > resourcesPerCatchpointFileChunkBackwardCompatible { + err = msgp.ErrOverflow(uint64(zb0005), uint64(resourcesPerCatchpointFileChunkBackwardCompatible)) + err = msgp.WrapError(err, "struct-from-array", "Resources") return } - } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).Manager.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Manager") - return + if zb0006 { + (*z).Resources = nil + } else if (*z).Resources == nil { + (*z).Resources = make(map[uint64]msgp.Raw, zb0005) } - } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).Reserve.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Reserve") - return + for zb0005 > 0 { + var zb0001 uint64 + var zb0002 msgp.Raw + zb0005-- + zb0001, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Resources") + return + } + bts, err = zb0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Resources", zb0001) + return + } + (*z).Resources[zb0001] = zb0002 } } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).Freeze.UnmarshalMsg(bts) + if zb0003 > 0 { + zb0003-- + (*z).ExpectingMoreEntries, bts, err = msgp.ReadBoolBytes(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Freeze") + err = msgp.WrapError(err, "struct-from-array", "ExpectingMoreEntries") return } } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).Clawback.UnmarshalMsg(bts) + if zb0003 > 0 { + err = msgp.ErrTooManyArrayFields(zb0003) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Clawback") + err = msgp.WrapError(err, "struct-from-array") return } } - if zb0002 > 0 { - zb0002-- - (*z).Amount, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Amount") - return - } + } else { + if err != nil { + err = msgp.WrapError(err) + return } - if zb0002 > 0 { - zb0002-- - (*z).Frozen, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Frozen") - return - } + if zb0004 { + (*z) = encodedBalanceRecordV6{} } - if zb0002 > 0 { - zb0002-- - (*z).SchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SchemaNumUint") + err = msgp.WrapError(err) return } - } - if zb0002 > 0 { - zb0002-- - (*z).SchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SchemaNumByteSlice") - return + switch string(field) { + case "a": + bts, err = (*z).Address.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Address") + return + } + case "b": + bts, err = (*z).AccountData.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "AccountData") + return + } + case "c": + var zb0007 int + var zb0008 bool + zb0007, zb0008, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Resources") + return + } + if zb0007 > resourcesPerCatchpointFileChunkBackwardCompatible { + err = msgp.ErrOverflow(uint64(zb0007), uint64(resourcesPerCatchpointFileChunkBackwardCompatible)) + err = msgp.WrapError(err, "Resources") + return + } + if zb0008 { + (*z).Resources = nil + } else if (*z).Resources == nil { + (*z).Resources = make(map[uint64]msgp.Raw, zb0007) + } + for zb0007 > 0 { + var zb0001 uint64 + var zb0002 msgp.Raw + zb0007-- + zb0001, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Resources") + return + } + bts, err = zb0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Resources", zb0001) + return + } + (*z).Resources[zb0001] = zb0002 + } + case "e": + (*z).ExpectingMoreEntries, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExpectingMoreEntries") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } } } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).KeyValue.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "KeyValue") - return - } + } + o = bts + return +} + +func (_ *encodedBalanceRecordV6) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedBalanceRecordV6) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *encodedBalanceRecordV6) Msgsize() (s int) { + s = 1 + 2 + (*z).Address.Msgsize() + 2 + (*z).AccountData.Msgsize() + 2 + msgp.MapHeaderSize + if (*z).Resources != nil { + for zb0001, zb0002 := range (*z).Resources { + _ = zb0001 + _ = zb0002 + s += 0 + msgp.Uint64Size + zb0002.Msgsize() } - if zb0002 > 0 { - zb0002-- - var zb0004 int - zb0004, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ApprovalProgram") - return - } - if zb0004 > config.MaxAvailableAppProgramLen { - err = msgp.ErrOverflow(uint64(zb0004), uint64(config.MaxAvailableAppProgramLen)) - return - } - (*z).ApprovalProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ApprovalProgram) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ApprovalProgram") - return - } + } + s += 2 + msgp.BoolSize + return +} + +// MsgIsZero returns whether this is a zero value +func (z *encodedBalanceRecordV6) MsgIsZero() bool { + return ((*z).Address.MsgIsZero()) && ((*z).AccountData.MsgIsZero()) && (len((*z).Resources) == 0) && ((*z).ExpectingMoreEntries == false) +} + +// MarshalMsg implements msgp.Marshaler +func (z *encodedKVRecordV6) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if len((*z).Key) == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + if len((*z).Value) == 0 { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "k" + o = append(o, 0xa1, 0x6b) + o = msgp.AppendBytes(o, (*z).Key) } - if zb0002 > 0 { - zb0002-- - var zb0005 int - zb0005, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ClearStateProgram") - return - } - if zb0005 > config.MaxAvailableAppProgramLen { - err = msgp.ErrOverflow(uint64(zb0005), uint64(config.MaxAvailableAppProgramLen)) - return - } - (*z).ClearStateProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ClearStateProgram) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ClearStateProgram") - return - } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendBytes(o, (*z).Value) } - if zb0002 > 0 { - zb0002-- - bts, err = (*z).GlobalState.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "GlobalState") - return - } + } + return +} + +func (_ *encodedKVRecordV6) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedKVRecordV6) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *encodedKVRecordV6) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return } - if zb0002 > 0 { - zb0002-- - (*z).LocalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if zb0001 > 0 { + zb0001-- + var zb0003 int + zb0003, err = msgp.ReadBytesBytesHeader(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "LocalStateSchemaNumUint") + err = msgp.WrapError(err, "struct-from-array", "Key") return } - } - if zb0002 > 0 { - zb0002-- - (*z).LocalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "LocalStateSchemaNumByteSlice") + if zb0003 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0003), uint64(encodedKVRecordV6MaxKeyLength)) return } - } - if zb0002 > 0 { - zb0002-- - (*z).GlobalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + (*z).Key, bts, err = msgp.ReadBytesBytes(bts, (*z).Key) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "GlobalStateSchemaNumUint") + err = msgp.WrapError(err, "struct-from-array", "Key") return } } - if zb0002 > 0 { - zb0002-- - (*z).GlobalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if zb0001 > 0 { + zb0001-- + var zb0004 int + zb0004, err = msgp.ReadBytesBytesHeader(bts) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "GlobalStateSchemaNumByteSlice") + err = msgp.WrapError(err, "struct-from-array", "Value") return } - } - if zb0002 > 0 { - zb0002-- - (*z).ExtraProgramPages, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ExtraProgramPages") + if zb0004 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0004), uint64(encodedKVRecordV6MaxValueLength)) return } - } - if zb0002 > 0 { - zb0002-- - { - var zb0006 uint8 - zb0006, bts, err = msgp.ReadUint8Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "ResourceFlags") - return - } - (*z).ResourceFlags = resourceFlags(zb0006) - } - } - if zb0002 > 0 { - zb0002-- - (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + (*z).Value, bts, err = msgp.ReadBytesBytes(bts, (*z).Value) if err != nil { - err = msgp.WrapError(err, "struct-from-array", "UpdateRound") + err = msgp.WrapError(err, "struct-from-array", "Value") return } } - if zb0002 > 0 { - err = msgp.ErrTooManyArrayFields(zb0002) + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { err = msgp.WrapError(err, "struct-from-array") return @@ -3732,195 +1964,47 @@ func (z *resourcesData) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0003 { - (*z) = resourcesData{} + if zb0002 { + (*z) = encodedKVRecordV6{} } - for zb0002 > 0 { - zb0002-- + for zb0001 > 0 { + zb0001-- field, bts, err = msgp.ReadMapKeyZC(bts) if err != nil { err = msgp.WrapError(err) return } switch string(field) { - case "a": - (*z).Total, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Total") - return - } - case "b": - (*z).Decimals, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Decimals") - return - } - case "c": - (*z).DefaultFrozen, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "DefaultFrozen") - return - } - case "d": - (*z).UnitName, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "UnitName") - return - } - case "e": - (*z).AssetName, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "AssetName") - return - } - case "f": - (*z).URL, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "URL") - return - } - case "g": - bts, err = msgp.ReadExactBytes(bts, ((*z).MetadataHash)[:]) - if err != nil { - err = msgp.WrapError(err, "MetadataHash") - return - } - case "h": - bts, err = (*z).Manager.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Manager") - return - } - case "i": - bts, err = (*z).Reserve.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Reserve") - return - } - case "j": - bts, err = (*z).Freeze.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Freeze") - return - } case "k": - bts, err = (*z).Clawback.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Clawback") - return - } - case "l": - (*z).Amount, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "Amount") - return - } - case "m": - (*z).Frozen, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Frozen") - return - } - case "n": - (*z).SchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "SchemaNumUint") - return - } - case "o": - (*z).SchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "SchemaNumByteSlice") - return - } - case "p": - bts, err = (*z).KeyValue.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "KeyValue") - return - } - case "q": - var zb0007 int - zb0007, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "ApprovalProgram") - return - } - if zb0007 > config.MaxAvailableAppProgramLen { - err = msgp.ErrOverflow(uint64(zb0007), uint64(config.MaxAvailableAppProgramLen)) - return - } - (*z).ApprovalProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ApprovalProgram) - if err != nil { - err = msgp.WrapError(err, "ApprovalProgram") - return - } - case "r": - var zb0008 int - zb0008, err = msgp.ReadBytesBytesHeader(bts) - if err != nil { - err = msgp.WrapError(err, "ClearStateProgram") - return - } - if zb0008 > config.MaxAvailableAppProgramLen { - err = msgp.ErrOverflow(uint64(zb0008), uint64(config.MaxAvailableAppProgramLen)) - return - } - (*z).ClearStateProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ClearStateProgram) - if err != nil { - err = msgp.WrapError(err, "ClearStateProgram") - return - } - case "s": - bts, err = (*z).GlobalState.UnmarshalMsg(bts) + var zb0005 int + zb0005, err = msgp.ReadBytesBytesHeader(bts) if err != nil { - err = msgp.WrapError(err, "GlobalState") + err = msgp.WrapError(err, "Key") return } - case "t": - (*z).LocalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "LocalStateSchemaNumUint") + if zb0005 > encodedKVRecordV6MaxKeyLength { + err = msgp.ErrOverflow(uint64(zb0005), uint64(encodedKVRecordV6MaxKeyLength)) return } - case "u": - (*z).LocalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + (*z).Key, bts, err = msgp.ReadBytesBytes(bts, (*z).Key) if err != nil { - err = msgp.WrapError(err, "LocalStateSchemaNumByteSlice") + err = msgp.WrapError(err, "Key") return } case "v": - (*z).GlobalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "GlobalStateSchemaNumUint") - return - } - case "w": - (*z).GlobalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + var zb0006 int + zb0006, err = msgp.ReadBytesBytesHeader(bts) if err != nil { - err = msgp.WrapError(err, "GlobalStateSchemaNumByteSlice") + err = msgp.WrapError(err, "Value") return } - case "x": - (*z).ExtraProgramPages, bts, err = msgp.ReadUint32Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "ExtraProgramPages") + if zb0006 > encodedKVRecordV6MaxValueLength { + err = msgp.ErrOverflow(uint64(zb0006), uint64(encodedKVRecordV6MaxValueLength)) return } - case "y": - { - var zb0009 uint8 - zb0009, bts, err = msgp.ReadUint8Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "ResourceFlags") - return - } - (*z).ResourceFlags = resourceFlags(zb0009) - } - case "z": - (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + (*z).Value, bts, err = msgp.ReadBytesBytes(bts, (*z).Value) if err != nil { - err = msgp.WrapError(err, "UpdateRound") + err = msgp.WrapError(err, "Value") return } default: @@ -3936,20 +2020,66 @@ func (z *resourcesData) UnmarshalMsg(bts []byte) (o []byte, err error) { return } -func (_ *resourcesData) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*resourcesData) +func (_ *encodedKVRecordV6) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*encodedKVRecordV6) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *encodedKVRecordV6) Msgsize() (s int) { + s = 1 + 2 + msgp.BytesPrefixSize + len((*z).Key) + 2 + msgp.BytesPrefixSize + len((*z).Value) + return +} + +// MsgIsZero returns whether this is a zero value +func (z *encodedKVRecordV6) MsgIsZero() bool { + return (len((*z).Key) == 0) && (len((*z).Value) == 0) +} + +// MarshalMsg implements msgp.Marshaler +func (z hashKind) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendByte(o, byte(z)) + return +} + +func (_ hashKind) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(hashKind) + if !ok { + _, ok = (z).(*hashKind) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *hashKind) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 byte + zb0001, bts, err = msgp.ReadByteBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = hashKind(zb0001) + } + o = bts + return +} + +func (_ *hashKind) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*hashKind) return ok } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *resourcesData) Msgsize() (s int) { - s = 3 + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.BoolSize + 2 + msgp.StringPrefixSize + len((*z).UnitName) + 2 + msgp.StringPrefixSize + len((*z).AssetName) + 2 + msgp.StringPrefixSize + len((*z).URL) + 2 + msgp.ArrayHeaderSize + (32 * (msgp.ByteSize)) + 2 + (*z).Manager.Msgsize() + 2 + (*z).Reserve.Msgsize() + 2 + (*z).Freeze.Msgsize() + 2 + (*z).Clawback.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.BoolSize + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + (*z).KeyValue.Msgsize() + 2 + msgp.BytesPrefixSize + len((*z).ApprovalProgram) + 2 + msgp.BytesPrefixSize + len((*z).ClearStateProgram) + 2 + (*z).GlobalState.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.Uint8Size + 2 + msgp.Uint64Size +func (z hashKind) Msgsize() (s int) { + s = msgp.ByteSize return } // MsgIsZero returns whether this is a zero value -func (z *resourcesData) MsgIsZero() bool { - return ((*z).Total == 0) && ((*z).Decimals == 0) && ((*z).DefaultFrozen == false) && ((*z).UnitName == "") && ((*z).AssetName == "") && ((*z).URL == "") && ((*z).MetadataHash == ([32]byte{})) && ((*z).Manager.MsgIsZero()) && ((*z).Reserve.MsgIsZero()) && ((*z).Freeze.MsgIsZero()) && ((*z).Clawback.MsgIsZero()) && ((*z).Amount == 0) && ((*z).Frozen == false) && ((*z).SchemaNumUint == 0) && ((*z).SchemaNumByteSlice == 0) && ((*z).KeyValue.MsgIsZero()) && (len((*z).ApprovalProgram) == 0) && (len((*z).ClearStateProgram) == 0) && ((*z).GlobalState.MsgIsZero()) && ((*z).LocalStateSchemaNumUint == 0) && ((*z).LocalStateSchemaNumByteSlice == 0) && ((*z).GlobalStateSchemaNumUint == 0) && ((*z).GlobalStateSchemaNumByteSlice == 0) && ((*z).ExtraProgramPages == 0) && ((*z).ResourceFlags == 0) && ((*z).UpdateRound == 0) +func (z hashKind) MsgIsZero() bool { + return z == 0 } // MarshalMsg implements msgp.Marshaler diff --git a/ledger/msgp_gen_test.go b/ledger/msgp_gen_test.go index 2481023983..2a03d80bc4 100644 --- a/ledger/msgp_gen_test.go +++ b/ledger/msgp_gen_test.go @@ -74,186 +74,6 @@ func BenchmarkUnmarshalCatchpointFileHeader(b *testing.B) { } } -func TestMarshalUnmarshalbaseAccountData(t *testing.T) { - partitiontest.PartitionTest(t) - v := baseAccountData{} - bts := v.MarshalMsg(nil) - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func TestRandomizedEncodingbaseAccountData(t *testing.T) { - protocol.RunEncodingTest(t, &baseAccountData{}) -} - -func BenchmarkMarshalMsgbaseAccountData(b *testing.B) { - v := baseAccountData{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgbaseAccountData(b *testing.B) { - v := baseAccountData{} - bts := make([]byte, 0, v.Msgsize()) - bts = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalbaseAccountData(b *testing.B) { - v := baseAccountData{} - bts := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestMarshalUnmarshalbaseOnlineAccountData(t *testing.T) { - partitiontest.PartitionTest(t) - v := baseOnlineAccountData{} - bts := v.MarshalMsg(nil) - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func TestRandomizedEncodingbaseOnlineAccountData(t *testing.T) { - protocol.RunEncodingTest(t, &baseOnlineAccountData{}) -} - -func BenchmarkMarshalMsgbaseOnlineAccountData(b *testing.B) { - v := baseOnlineAccountData{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgbaseOnlineAccountData(b *testing.B) { - v := baseOnlineAccountData{} - bts := make([]byte, 0, v.Msgsize()) - bts = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalbaseOnlineAccountData(b *testing.B) { - v := baseOnlineAccountData{} - bts := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestMarshalUnmarshalbaseVotingData(t *testing.T) { - partitiontest.PartitionTest(t) - v := baseVotingData{} - bts := v.MarshalMsg(nil) - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func TestRandomizedEncodingbaseVotingData(t *testing.T) { - protocol.RunEncodingTest(t, &baseVotingData{}) -} - -func BenchmarkMarshalMsgbaseVotingData(b *testing.B) { - v := baseVotingData{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgbaseVotingData(b *testing.B) { - v := baseVotingData{} - bts := make([]byte, 0, v.Msgsize()) - bts = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalbaseVotingData(b *testing.B) { - v := baseVotingData{} - bts := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshalcatchpointFileBalancesChunkV5(t *testing.T) { partitiontest.PartitionTest(t) v := catchpointFileBalancesChunkV5{} @@ -614,66 +434,6 @@ func BenchmarkUnmarshalencodedKVRecordV6(b *testing.B) { } } -func TestMarshalUnmarshalresourcesData(t *testing.T) { - partitiontest.PartitionTest(t) - v := resourcesData{} - bts := v.MarshalMsg(nil) - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func TestRandomizedEncodingresourcesData(t *testing.T) { - protocol.RunEncodingTest(t, &resourcesData{}) -} - -func BenchmarkMarshalMsgresourcesData(b *testing.B) { - v := resourcesData{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgresourcesData(b *testing.B) { - v := resourcesData{} - bts := make([]byte, 0, v.Msgsize()) - bts = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalresourcesData(b *testing.B) { - v := resourcesData{} - bts := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshaltxTailRound(t *testing.T) { partitiontest.PartitionTest(t) v := txTailRound{} diff --git a/ledger/onlineaccountscache.go b/ledger/onlineaccountscache.go index 2d43994094..4d6c976027 100644 --- a/ledger/onlineaccountscache.go +++ b/ledger/onlineaccountscache.go @@ -20,6 +20,7 @@ import ( "container/list" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" ) // Worst case memory usage = 2500 * 320 * 150B = 120MB @@ -34,17 +35,17 @@ type onlineAccountsCache struct { // init initializes the onlineAccountsCache for use. // thread locking semantics : write lock -func (o *onlineAccountsCache) init(accts []persistedOnlineAccountData, maxCacheSize int) { +func (o *onlineAccountsCache) init(accts []store.PersistedOnlineAccountData, maxCacheSize int) { o.accounts = make(map[basics.Address]*list.List) o.maxCacheSize = maxCacheSize for _, acct := range accts { // if cache full, stop writing cachedAcct := cachedOnlineAccount{ - baseOnlineAccountData: acct.accountData, - updRound: acct.updRound, + BaseOnlineAccountData: acct.AccountData, + updRound: acct.UpdRound, } - if !o.writeFront(acct.addr, cachedAcct) { + if !o.writeFront(acct.Addr, cachedAcct) { break } } diff --git a/ledger/onlineaccountscache_test.go b/ledger/onlineaccountscache_test.go index 42a94bb42c..23010b35d8 100644 --- a/ledger/onlineaccountscache_test.go +++ b/ledger/onlineaccountscache_test.go @@ -24,6 +24,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" @@ -44,7 +45,7 @@ func TestOnlineAccountsCacheBasic(t *testing.T) { for i := 0; i < roundsNum; i++ { acct := cachedOnlineAccount{ updRound: basics.Round(i), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } written := oac.writeFront(addr, acct) require.True(t, written) @@ -61,7 +62,7 @@ func TestOnlineAccountsCacheBasic(t *testing.T) { for i := proto.MaxBalLookback; i < uint64(roundsNum)+proto.MaxBalLookback; i++ { acct := cachedOnlineAccount{ updRound: basics.Round(i), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: i}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: i}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } written := oac.writeFront(addr, acct) require.True(t, written) @@ -88,7 +89,7 @@ func TestOnlineAccountsCacheBasic(t *testing.T) { // attempt to insert a value with the updRound less than latest, expect it to have ignored acct = cachedOnlineAccount{ updRound: basics.Round(uint64(roundsNum) + proto.MaxBalLookback - 1), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: 100}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: 100}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } written := oac.writeFront(addr, acct) require.False(t, written) @@ -109,13 +110,13 @@ func TestOnlineAccountsCachePruneOffline(t *testing.T) { for i := 0; i < roundsNum; i++ { acct := cachedOnlineAccount{ updRound: basics.Round(i), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } oac.writeFront(addr, acct) } acct := cachedOnlineAccount{ updRound: basics.Round(roundsNum), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(roundsNum)}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(roundsNum)}}, } oac.writeFront(addr, acct) @@ -139,7 +140,7 @@ func TestOnlineAccountsCacheMaxEntries(t *testing.T) { lastAddr = ledgertesting.RandomAddress() acct := cachedOnlineAccount{ updRound: basics.Round(i), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(i)}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } written := oac.writeFront(lastAddr, acct) require.True(t, written) @@ -147,7 +148,7 @@ func TestOnlineAccountsCacheMaxEntries(t *testing.T) { acct := cachedOnlineAccount{ updRound: basics.Round(100), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, baseVotingData: baseVotingData{VoteLastValid: 1000}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, BaseVotingData: store.BaseVotingData{VoteLastValid: 1000}}, } written := oac.writeFront(ledgertesting.RandomAddress(), acct) require.False(t, written) @@ -158,7 +159,7 @@ func TestOnlineAccountsCacheMaxEntries(t *testing.T) { // set one to be expired acct = cachedOnlineAccount{ updRound: basics.Round(maxCacheSize), - baseOnlineAccountData: baseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, baseVotingData: baseVotingData{}}, + BaseOnlineAccountData: store.BaseOnlineAccountData{MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, BaseVotingData: store.BaseVotingData{}}, } written = oac.writeFront(lastAddr, acct) require.True(t, written) diff --git a/ledger/persistedaccts_list.go b/ledger/persistedaccts_list.go index de8c8380fa..c7c884a860 100644 --- a/ledger/persistedaccts_list.go +++ b/ledger/persistedaccts_list.go @@ -16,6 +16,8 @@ package ledger +import "github.com/algorand/go-algorand/ledger/store" + // persistedAccountDataList represents a doubly linked list. // must initiate with newPersistedAccountList. type persistedAccountDataList struct { @@ -31,7 +33,7 @@ type persistedAccountDataListNode struct { // element (l.Front()). next, prev *persistedAccountDataListNode - Value *persistedAccountData + Value *store.PersistedAccountData } func newPersistedAccountList() *persistedAccountDataList { @@ -99,7 +101,7 @@ func (l *persistedAccountDataList) remove(e *persistedAccountDataListNode) { } // pushFront inserts a new element e with value v at the front of list l and returns e. -func (l *persistedAccountDataList) pushFront(v *persistedAccountData) *persistedAccountDataListNode { +func (l *persistedAccountDataList) pushFront(v *store.PersistedAccountData) *persistedAccountDataListNode { newNode := l.getNewNode() newNode.Value = v return l.insertValue(newNode, &l.root) diff --git a/ledger/persistedaccts_list_test.go b/ledger/persistedaccts_list_test.go index 8e8b9531b6..9c7b0d69ce 100644 --- a/ledger/persistedaccts_list_test.go +++ b/ledger/persistedaccts_list_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -77,9 +78,9 @@ func checkListLen(t *testing.T, l dataList, len int) bool { func TestRemoveFromListAD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedAccountList() - e1 := l.pushFront(&persistedAccountData{addr: basics.Address{1}}) - e2 := l.pushFront(&persistedAccountData{addr: basics.Address{2}}) - e3 := l.pushFront(&persistedAccountData{addr: basics.Address{3}}) + e1 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{1}}) + e2 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{2}}) + e3 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{3}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e3, e2, e1}) l.remove(e2) @@ -97,7 +98,7 @@ func TestAddingNewNodeWithAllocatedFreeListAD(t *testing.T) { return } // test elements - e1 := l.pushFront(&persistedAccountData{addr: basics.Address{1}}) + e1 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{1}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e1}) if countListSize(l.freeList) != 9 { @@ -167,11 +168,11 @@ func TestMultielementListPositioningAD(t *testing.T) { l := newPersistedAccountList() checkListPointersAD(t, l, []*persistedAccountDataListNode{}) // test elements - e2 := l.pushFront(&persistedAccountData{addr: basics.Address{2}}) - e1 := l.pushFront(&persistedAccountData{addr: basics.Address{1}}) - e3 := l.pushFront(&persistedAccountData{addr: basics.Address{3}}) - e4 := l.pushFront(&persistedAccountData{addr: basics.Address{4}}) - e5 := l.pushFront(&persistedAccountData{addr: basics.Address{5}}) + e2 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{2}}) + e1 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{1}}) + e3 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{3}}) + e4 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{4}}) + e5 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{5}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e5, e4, e3, e1, e2}) @@ -199,7 +200,7 @@ func TestMultielementListPositioningAD(t *testing.T) { l.moveToFront(e1) // no movement checkListPointersAD(t, l, []*persistedAccountDataListNode{e1, e3, e4}) - e2 = l.pushFront(&persistedAccountData{addr: basics.Address{2}}) + e2 = l.pushFront(&store.PersistedAccountData{Addr: basics.Address{2}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e2, e1, e3, e4}) l.remove(e3) // removing from middle @@ -225,7 +226,7 @@ func TestSingleElementListPositioningAD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedAccountList() checkListPointersAD(t, l, []*persistedAccountDataListNode{}) - e := l.pushFront(&persistedAccountData{addr: basics.Address{1}}) + e := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{1}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e}) l.moveToFront(e) checkListPointersAD(t, l, []*persistedAccountDataListNode{e}) @@ -236,8 +237,8 @@ func TestSingleElementListPositioningAD(t *testing.T) { func TestRemovedNodeShouldBeMovedToFreeListAD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedAccountList() - e1 := l.pushFront(&persistedAccountData{addr: basics.Address{1}}) - e2 := l.pushFront(&persistedAccountData{addr: basics.Address{2}}) + e1 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{1}}) + e2 := l.pushFront(&store.PersistedAccountData{Addr: basics.Address{2}}) checkListPointersAD(t, l, []*persistedAccountDataListNode{e2, e1}) diff --git a/ledger/persistedonlineaccts_list.go b/ledger/persistedonlineaccts_list.go index a5c37ca8fb..e5775498d1 100644 --- a/ledger/persistedonlineaccts_list.go +++ b/ledger/persistedonlineaccts_list.go @@ -16,6 +16,8 @@ package ledger +import "github.com/algorand/go-algorand/ledger/store" + // persistedOnlineAccountDataList represents a doubly linked list. // must initiate with newPersistedAccountList. type persistedOnlineAccountDataList struct { @@ -31,7 +33,7 @@ type persistedOnlineAccountDataListNode struct { // element (l.Front()). next, prev *persistedOnlineAccountDataListNode - Value *persistedOnlineAccountData + Value *store.PersistedOnlineAccountData } func newPersistedOnlineAccountList() *persistedOnlineAccountDataList { @@ -99,7 +101,7 @@ func (l *persistedOnlineAccountDataList) remove(e *persistedOnlineAccountDataLis } // pushFront inserts a new element e with value v at the front of list l and returns e. -func (l *persistedOnlineAccountDataList) pushFront(v *persistedOnlineAccountData) *persistedOnlineAccountDataListNode { +func (l *persistedOnlineAccountDataList) pushFront(v *store.PersistedOnlineAccountData) *persistedOnlineAccountDataListNode { newNode := l.getNewNode() newNode.Value = v return l.insertValue(newNode, &l.root) diff --git a/ledger/persistedonlineaccts_list_test.go b/ledger/persistedonlineaccts_list_test.go index 7bc0ad3733..7e4a10721b 100644 --- a/ledger/persistedonlineaccts_list_test.go +++ b/ledger/persistedonlineaccts_list_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -45,9 +46,9 @@ func (l *persistedOnlineAccountDataListNode) getPrev() dataListNode { func TestRemoveFromListOAD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedOnlineAccountList() - e1 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{1}}) - e2 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{2}}) - e3 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{3}}) + e1 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{1}}) + e2 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{2}}) + e3 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{3}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e3, e2, e1}) l.remove(e2) @@ -65,7 +66,7 @@ func TestAddingNewNodeWithAllocatedFreeListOAD(t *testing.T) { return } // test elements - e1 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{1}}) + e1 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{1}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e1}) if countListSize(l.freeList) != 9 { @@ -89,11 +90,11 @@ func TestMultielementListPositioningOAD(t *testing.T) { l := newPersistedOnlineAccountList() checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{}) // test elements - e2 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{2}}) - e1 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{1}}) - e3 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{3}}) - e4 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{4}}) - e5 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{5}}) + e2 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{2}}) + e1 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{1}}) + e3 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{3}}) + e4 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{4}}) + e5 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{5}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e5, e4, e3, e1, e2}) @@ -121,7 +122,7 @@ func TestMultielementListPositioningOAD(t *testing.T) { l.moveToFront(e1) // no movement checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e1, e3, e4}) - e2 = l.pushFront(&persistedOnlineAccountData{addr: basics.Address{2}}) + e2 = l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{2}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e2, e1, e3, e4}) l.remove(e3) // removing from middle @@ -147,7 +148,7 @@ func TestSingleElementListPositioningOD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedOnlineAccountList() checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{}) - e := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{1}}) + e := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{1}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e}) l.moveToFront(e) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e}) @@ -158,8 +159,8 @@ func TestSingleElementListPositioningOD(t *testing.T) { func TestRemovedNodeShouldBeMovedToFreeListOAD(t *testing.T) { partitiontest.PartitionTest(t) l := newPersistedOnlineAccountList() - e1 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{1}}) - e2 := l.pushFront(&persistedOnlineAccountData{addr: basics.Address{2}}) + e1 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{1}}) + e2 := l.pushFront(&store.PersistedOnlineAccountData{Addr: basics.Address{2}}) checkListPointersOAD(t, l, []*persistedOnlineAccountDataListNode{e2, e1}) diff --git a/ledger/store/data.go b/ledger/store/data.go new file mode 100644 index 0000000000..006f43bead --- /dev/null +++ b/ledger/store/data.go @@ -0,0 +1,760 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package store + +import ( + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklesignature" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" +) + +// BaseAccountData is the base struct used to store account data +type BaseAccountData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Status basics.Status `codec:"a"` + MicroAlgos basics.MicroAlgos `codec:"b"` + RewardsBase uint64 `codec:"c"` + RewardedMicroAlgos basics.MicroAlgos `codec:"d"` + AuthAddr basics.Address `codec:"e"` + TotalAppSchemaNumUint uint64 `codec:"f"` + TotalAppSchemaNumByteSlice uint64 `codec:"g"` + TotalExtraAppPages uint32 `codec:"h"` + TotalAssetParams uint64 `codec:"i"` + TotalAssets uint64 `codec:"j"` + TotalAppParams uint64 `codec:"k"` + TotalAppLocalStates uint64 `codec:"l"` + TotalBoxes uint64 `codec:"m"` + TotalBoxBytes uint64 `codec:"n"` + + BaseVotingData + + // UpdateRound is the round that modified this account data last. Since we want all the nodes to have the exact same + // value for this field, we'll be setting the value of this field to zero *before* the EnableAccountDataResourceSeparation + // consensus parameter is being set. Once the above consensus takes place, this field would be populated with the + // correct round number. + UpdateRound uint64 `codec:"z"` +} + +// ResourceFlags are bitmask used to indicate which portions ofa resources are used. +type ResourceFlags uint8 + +const ( + // ResourceFlagsHolding indicates "Holding" + ResourceFlagsHolding ResourceFlags = 0 + // ResourceFlagsNotHolding indicates "Not Holding" + ResourceFlagsNotHolding ResourceFlags = 1 + // ResourceFlagsOwnership indicates "Ownerhip" + ResourceFlagsOwnership ResourceFlags = 2 + // ResourceFlagsEmptyAsset indicates "Empty Asset" + ResourceFlagsEmptyAsset ResourceFlags = 4 + // ResourceFlagsEmptyApp indicates "Empty App" + ResourceFlagsEmptyApp ResourceFlags = 8 +) + +// +// Resource flags interpretation: +// +// ResourceFlagsHolding - the resource contains the holding of asset/app. +// ResourceFlagsNotHolding - the resource is completely empty. This state should not be persisted. +// ResourceFlagsOwnership - the resource contains the asset parameter or application parameters. +// ResourceFlagsEmptyAsset - this is an asset resource, and it is empty. +// ResourceFlagsEmptyApp - this is an app resource, and it is empty. + +// ResourcesData holds the resource data that will be stored. +type ResourcesData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + // asset parameters ( basics.AssetParams ) + Total uint64 `codec:"a"` + Decimals uint32 `codec:"b"` + DefaultFrozen bool `codec:"c"` + UnitName string `codec:"d"` + AssetName string `codec:"e"` + URL string `codec:"f"` + MetadataHash [32]byte `codec:"g"` + Manager basics.Address `codec:"h"` + Reserve basics.Address `codec:"i"` + Freeze basics.Address `codec:"j"` + Clawback basics.Address `codec:"k"` + + // asset holding ( basics.AssetHolding ) + Amount uint64 `codec:"l"` + Frozen bool `codec:"m"` + + // application local state ( basics.AppLocalState ) + SchemaNumUint uint64 `codec:"n"` + SchemaNumByteSlice uint64 `codec:"o"` + KeyValue basics.TealKeyValue `codec:"p"` + + // application global params ( basics.AppParams ) + ApprovalProgram []byte `codec:"q,allocbound=config.MaxAvailableAppProgramLen"` + ClearStateProgram []byte `codec:"r,allocbound=config.MaxAvailableAppProgramLen"` + GlobalState basics.TealKeyValue `codec:"s"` + LocalStateSchemaNumUint uint64 `codec:"t"` + LocalStateSchemaNumByteSlice uint64 `codec:"u"` + GlobalStateSchemaNumUint uint64 `codec:"v"` + GlobalStateSchemaNumByteSlice uint64 `codec:"w"` + ExtraProgramPages uint32 `codec:"x"` + + // ResourceFlags helps to identify which portions of this structure should be used; in particular, it + // helps to provide a marker - i.e. whether the account was, for instance, opted-in for the asset compared + // to just being the owner of the asset. A comparison against the empty structure doesn't work here - + // since both the holdings and the parameters are allowed to be all at their default values. + ResourceFlags ResourceFlags `codec:"y"` + + // UpdateRound is the round that modified this resource last. Since we want all the nodes to have the exact same + // value for this field, we'll be setting the value of this field to zero *before* the EnableAccountDataResourceSeparation + // consensus parameter is being set. Once the above consensus takes place, this field would be populated with the + // correct round number. + UpdateRound uint64 `codec:"z"` +} + +// BaseVotingData is the base struct used to store voting data +type BaseVotingData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + VoteID crypto.OneTimeSignatureVerifier `codec:"A"` + SelectionID crypto.VRFVerifier `codec:"B"` + VoteFirstValid basics.Round `codec:"C"` + VoteLastValid basics.Round `codec:"D"` + VoteKeyDilution uint64 `codec:"E"` + StateProofID merklesignature.Commitment `codec:"F"` +} + +// BaseOnlineAccountData is the base struct used to store online account data +type BaseOnlineAccountData struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + BaseVotingData + + MicroAlgos basics.MicroAlgos `codec:"Y"` + RewardsBase uint64 `codec:"Z"` +} + +// PersistedKVData represents the stored entry behind a application boxed key/value. +type PersistedKVData struct { + // kv value + Value []byte + // the round number that is associated with the kv value. This field is the corresponding one to the round field + // in persistedAccountData, and serves the same purpose. + Round basics.Round +} + +// PersistedAccountData is used for representing a single account stored on the disk. In addition to the +// basics.AccountData, it also stores complete referencing information used to maintain the base accounts +// list. +type PersistedAccountData struct { + // The address of the account. In contrasts to maps, having this value explicitly here allows us to use this + // data structure in queues directly, without "attaching" the address as the address as the map key. + Addr basics.Address + // The underlaying account data + AccountData BaseAccountData + // The rowid, when available. If the entry was loaded from the disk, then we have the rowid for it. Entries + // that doesn't have rowid ( hence, rowid == 0 ) represent either deleted accounts or non-existing accounts. + Rowid int64 + // the round number that is associated with the accountData. This field is needed so that we can maintain a correct + // lruAccounts cache. We use it to ensure that the entries on the lruAccounts.accountsList are the latest ones. + // this becomes an issue since while we attempt to write an update to disk, we might be reading an entry and placing + // it on the lruAccounts.pendingAccounts; The commitRound doesn't attempt to flush the pending accounts, but rather + // just write the latest ( which is correct ) to the lruAccounts.accountsList. later on, during on newBlockImpl, we + // want to ensure that the "real" written value isn't being overridden by the value from the pending accounts. + Round basics.Round +} + +// PersistedResourcesData is exported view of persistedResourcesData +type PersistedResourcesData struct { + // addrid is the rowid of the account address that holds this resource. + // it is used in update/delete operations so must be filled for existing records. + // resolution is a multi stage process: + // - baseResources cache might have valid entries + // - baseAccount cache might have an entry for the address with rowid set + // - when loading non-cached resources in resourcesLoadOld + // - when creating new accounts in accountsNewRound + Addrid int64 + // creatable index + Aidx basics.CreatableIndex + // actual resource data + Data ResourcesData + // the round number that is associated with the resourcesData. This field is the corresponding one to the round field + // in persistedAccountData, and serves the same purpose. + Round basics.Round +} + +// PersistedOnlineAccountData is exported view of persistedOnlineAccountData +type PersistedOnlineAccountData struct { + Addr basics.Address + AccountData BaseOnlineAccountData + Rowid int64 + // the round number that is associated with the baseOnlineAccountData. This field is the corresponding one to the round field + // in persistedAccountData, and serves the same purpose. This value comes from account rounds table and correspond to + // the last trackers db commit round. + Round basics.Round + // the round number that the online account is for, i.e. account state change round. + UpdRound basics.Round +} + +// AccountResource returns the corresponding account resource data based on the type of resource. +func (prd *PersistedResourcesData) AccountResource() ledgercore.AccountResource { + var ret ledgercore.AccountResource + if prd.Data.IsAsset() { + if prd.Data.IsHolding() { + holding := prd.Data.GetAssetHolding() + ret.AssetHolding = &holding + } + if prd.Data.IsOwning() { + assetParams := prd.Data.GetAssetParams() + ret.AssetParams = &assetParams + } + } + if prd.Data.IsApp() { + if prd.Data.IsHolding() { + localState := prd.Data.GetAppLocalState() + ret.AppLocalState = &localState + } + if prd.Data.IsOwning() { + appParams := prd.Data.GetAppParams() + ret.AppParams = &appParams + } + } + return ret +} + +// NormalizedOnlineBalance getter for normalized online balance. +func (ba *BaseAccountData) NormalizedOnlineBalance(proto config.ConsensusParams) uint64 { + return basics.NormalizedOnlineAccountBalance(ba.Status, ba.RewardsBase, ba.MicroAlgos, proto) +} + +// SetCoreAccountData setter for core account data. +func (ba *BaseAccountData) SetCoreAccountData(ad *ledgercore.AccountData) { + ba.Status = ad.Status + ba.MicroAlgos = ad.MicroAlgos + ba.RewardsBase = ad.RewardsBase + ba.RewardedMicroAlgos = ad.RewardedMicroAlgos + ba.AuthAddr = ad.AuthAddr + ba.TotalAppSchemaNumUint = ad.TotalAppSchema.NumUint + ba.TotalAppSchemaNumByteSlice = ad.TotalAppSchema.NumByteSlice + ba.TotalExtraAppPages = ad.TotalExtraAppPages + ba.TotalAssetParams = ad.TotalAssetParams + ba.TotalAssets = ad.TotalAssets + ba.TotalAppParams = ad.TotalAppParams + ba.TotalAppLocalStates = ad.TotalAppLocalStates + ba.TotalBoxes = ad.TotalBoxes + ba.TotalBoxBytes = ad.TotalBoxBytes + + ba.BaseVotingData.SetCoreAccountData(ad) +} + +// SetAccountData setter for account data. +func (ba *BaseAccountData) SetAccountData(ad *basics.AccountData) { + ba.Status = ad.Status + ba.MicroAlgos = ad.MicroAlgos + ba.RewardsBase = ad.RewardsBase + ba.RewardedMicroAlgos = ad.RewardedMicroAlgos + ba.AuthAddr = ad.AuthAddr + ba.TotalAppSchemaNumUint = ad.TotalAppSchema.NumUint + ba.TotalAppSchemaNumByteSlice = ad.TotalAppSchema.NumByteSlice + ba.TotalExtraAppPages = ad.TotalExtraAppPages + ba.TotalAssetParams = uint64(len(ad.AssetParams)) + ba.TotalAssets = uint64(len(ad.Assets)) + ba.TotalAppParams = uint64(len(ad.AppParams)) + ba.TotalAppLocalStates = uint64(len(ad.AppLocalStates)) + ba.TotalBoxes = ad.TotalBoxes + ba.TotalBoxBytes = ad.TotalBoxBytes + + ba.BaseVotingData.VoteID = ad.VoteID + ba.BaseVotingData.SelectionID = ad.SelectionID + ba.BaseVotingData.StateProofID = ad.StateProofID + ba.BaseVotingData.VoteFirstValid = ad.VoteFirstValid + ba.BaseVotingData.VoteLastValid = ad.VoteLastValid + ba.BaseVotingData.VoteKeyDilution = ad.VoteKeyDilution +} + +// GetLedgerCoreAccountData getter for account data. +func (ba *BaseAccountData) GetLedgerCoreAccountData() ledgercore.AccountData { + return ledgercore.AccountData{ + AccountBaseData: ba.GetLedgerCoreAccountBaseData(), + VotingData: ba.GetLedgerCoreVotingData(), + } +} + +// GetLedgerCoreAccountBaseData getter for account base data. +func (ba *BaseAccountData) GetLedgerCoreAccountBaseData() ledgercore.AccountBaseData { + return ledgercore.AccountBaseData{ + Status: ba.Status, + MicroAlgos: ba.MicroAlgos, + RewardsBase: ba.RewardsBase, + RewardedMicroAlgos: ba.RewardedMicroAlgos, + AuthAddr: ba.AuthAddr, + TotalAppSchema: basics.StateSchema{ + NumUint: ba.TotalAppSchemaNumUint, + NumByteSlice: ba.TotalAppSchemaNumByteSlice, + }, + TotalExtraAppPages: ba.TotalExtraAppPages, + TotalAppParams: ba.TotalAppParams, + TotalAppLocalStates: ba.TotalAppLocalStates, + TotalAssetParams: ba.TotalAssetParams, + TotalAssets: ba.TotalAssets, + TotalBoxes: ba.TotalBoxes, + TotalBoxBytes: ba.TotalBoxBytes, + } +} + +// GetLedgerCoreVotingData getter for voting data. +func (ba *BaseAccountData) GetLedgerCoreVotingData() ledgercore.VotingData { + return ledgercore.VotingData{ + VoteID: ba.VoteID, + SelectionID: ba.SelectionID, + StateProofID: ba.StateProofID, + VoteFirstValid: ba.VoteFirstValid, + VoteLastValid: ba.VoteLastValid, + VoteKeyDilution: ba.VoteKeyDilution, + } +} + +// GetAccountData getter for account data. +func (ba *BaseAccountData) GetAccountData() basics.AccountData { + return basics.AccountData{ + Status: ba.Status, + MicroAlgos: ba.MicroAlgos, + RewardsBase: ba.RewardsBase, + RewardedMicroAlgos: ba.RewardedMicroAlgos, + AuthAddr: ba.AuthAddr, + TotalAppSchema: basics.StateSchema{ + NumUint: ba.TotalAppSchemaNumUint, + NumByteSlice: ba.TotalAppSchemaNumByteSlice, + }, + TotalExtraAppPages: ba.TotalExtraAppPages, + TotalBoxes: ba.TotalBoxes, + TotalBoxBytes: ba.TotalBoxBytes, + + VoteID: ba.VoteID, + SelectionID: ba.SelectionID, + StateProofID: ba.StateProofID, + VoteFirstValid: ba.VoteFirstValid, + VoteLastValid: ba.VoteLastValid, + VoteKeyDilution: ba.VoteKeyDilution, + } +} + +// IsEmpty return true if any of the fields other then the UpdateRound are non-zero. +func (ba *BaseAccountData) IsEmpty() bool { + return ba.Status == 0 && + ba.MicroAlgos.Raw == 0 && + ba.RewardsBase == 0 && + ba.RewardedMicroAlgos.Raw == 0 && + ba.AuthAddr.IsZero() && + ba.TotalAppSchemaNumUint == 0 && + ba.TotalAppSchemaNumByteSlice == 0 && + ba.TotalExtraAppPages == 0 && + ba.TotalAssetParams == 0 && + ba.TotalAssets == 0 && + ba.TotalAppParams == 0 && + ba.TotalAppLocalStates == 0 && + ba.TotalBoxes == 0 && + ba.TotalBoxBytes == 0 && + ba.BaseVotingData.IsEmpty() +} + +// IsEmpty returns true if all of the fields are zero. +func (bv BaseVotingData) IsEmpty() bool { + return bv == BaseVotingData{} +} + +// SetCoreAccountData initializes baseVotingData from ledgercore.AccountData +func (bv *BaseVotingData) SetCoreAccountData(ad *ledgercore.AccountData) { + bv.VoteID = ad.VoteID + bv.SelectionID = ad.SelectionID + bv.StateProofID = ad.StateProofID + bv.VoteFirstValid = ad.VoteFirstValid + bv.VoteLastValid = ad.VoteLastValid + bv.VoteKeyDilution = ad.VoteKeyDilution +} + +// IsVotingEmpty checks if voting data fields are empty +func (bo *BaseOnlineAccountData) IsVotingEmpty() bool { + return bo.BaseVotingData.IsEmpty() +} + +// IsEmpty return true if any of the fields are non-zero. +func (bo *BaseOnlineAccountData) IsEmpty() bool { + return bo.IsVotingEmpty() && + bo.MicroAlgos.Raw == 0 && + bo.RewardsBase == 0 +} + +// GetOnlineAccount returns ledgercore.OnlineAccount for top online accounts / voters +// TODO: unify +func (bo *BaseOnlineAccountData) GetOnlineAccount(addr basics.Address, normBalance uint64) ledgercore.OnlineAccount { + return ledgercore.OnlineAccount{ + Address: addr, + MicroAlgos: bo.MicroAlgos, + RewardsBase: bo.RewardsBase, + NormalizedOnlineBalance: normBalance, + VoteFirstValid: bo.VoteFirstValid, + VoteLastValid: bo.VoteLastValid, + StateProofID: bo.StateProofID, + } +} + +// GetOnlineAccountData returns basics.OnlineAccountData for lookup agreement +// TODO: unify with GetOnlineAccount/ledgercore.OnlineAccount +func (bo *BaseOnlineAccountData) GetOnlineAccountData(proto config.ConsensusParams, rewardsLevel uint64) ledgercore.OnlineAccountData { + microAlgos, _, _ := basics.WithUpdatedRewards( + proto, basics.Online, bo.MicroAlgos, basics.MicroAlgos{}, bo.RewardsBase, rewardsLevel, + ) + + return ledgercore.OnlineAccountData{ + MicroAlgosWithRewards: microAlgos, + VotingData: ledgercore.VotingData{ + VoteID: bo.VoteID, + SelectionID: bo.SelectionID, + StateProofID: bo.StateProofID, + VoteFirstValid: bo.VoteFirstValid, + VoteLastValid: bo.VoteLastValid, + VoteKeyDilution: bo.VoteKeyDilution, + }, + } +} + +// NormalizedOnlineBalance getter for normalized online balance. +func (bo *BaseOnlineAccountData) NormalizedOnlineBalance(proto config.ConsensusParams) uint64 { + return basics.NormalizedOnlineAccountBalance(basics.Online, bo.RewardsBase, bo.MicroAlgos, proto) +} + +// SetCoreAccountData setter for core account data. +func (bo *BaseOnlineAccountData) SetCoreAccountData(ad *ledgercore.AccountData) { + bo.BaseVotingData.SetCoreAccountData(ad) + + // MicroAlgos/RewardsBase are updated by the evaluator when accounts are touched + bo.MicroAlgos = ad.MicroAlgos + bo.RewardsBase = ad.RewardsBase +} + +// MakeResourcesData returns a new empty instance of resourcesData. +// Using this constructor method is necessary because of the ResourceFlags field. +// An optional rnd args sets UpdateRound +func MakeResourcesData(rnd uint64) ResourcesData { + return ResourcesData{ResourceFlags: ResourceFlagsNotHolding, UpdateRound: rnd} +} + +// IsHolding returns true if the resource flag is ResourceFlagsHolding +func (rd *ResourcesData) IsHolding() bool { + return (rd.ResourceFlags & ResourceFlagsNotHolding) == ResourceFlagsHolding +} + +// IsOwning returns true if the resource flag is ResourceFlagsOwnership +func (rd *ResourcesData) IsOwning() bool { + return (rd.ResourceFlags & ResourceFlagsOwnership) == ResourceFlagsOwnership +} + +// IsEmpty returns true if the resource flag is not an app or asset. +func (rd *ResourcesData) IsEmpty() bool { + return !rd.IsApp() && !rd.IsAsset() +} + +// IsEmptyAppFields returns true if the app fields are empty. +func (rd *ResourcesData) IsEmptyAppFields() bool { + return rd.SchemaNumUint == 0 && + rd.SchemaNumByteSlice == 0 && + len(rd.KeyValue) == 0 && + len(rd.ApprovalProgram) == 0 && + len(rd.ClearStateProgram) == 0 && + len(rd.GlobalState) == 0 && + rd.LocalStateSchemaNumUint == 0 && + rd.LocalStateSchemaNumByteSlice == 0 && + rd.GlobalStateSchemaNumUint == 0 && + rd.GlobalStateSchemaNumByteSlice == 0 && + rd.ExtraProgramPages == 0 +} + +// IsApp returns true if the flag is ResourceFlagsEmptyApp and the fields are not empty. +func (rd *ResourcesData) IsApp() bool { + if (rd.ResourceFlags & ResourceFlagsEmptyApp) == ResourceFlagsEmptyApp { + return true + } + return !rd.IsEmptyAppFields() +} + +// IsEmptyAssetFields returns true if the asset fields are empty. +func (rd *ResourcesData) IsEmptyAssetFields() bool { + return rd.Amount == 0 && + !rd.Frozen && + rd.Total == 0 && + rd.Decimals == 0 && + !rd.DefaultFrozen && + rd.UnitName == "" && + rd.AssetName == "" && + rd.URL == "" && + rd.MetadataHash == [32]byte{} && + rd.Manager.IsZero() && + rd.Reserve.IsZero() && + rd.Freeze.IsZero() && + rd.Clawback.IsZero() +} + +// IsAsset returns true if the flag is ResourceFlagsEmptyAsset and the fields are not empty. +func (rd *ResourcesData) IsAsset() bool { + if (rd.ResourceFlags & ResourceFlagsEmptyAsset) == ResourceFlagsEmptyAsset { + return true + } + return !rd.IsEmptyAssetFields() +} + +// ClearAssetParams clears the asset params. +func (rd *ResourcesData) ClearAssetParams() { + rd.Total = 0 + rd.Decimals = 0 + rd.DefaultFrozen = false + rd.UnitName = "" + rd.AssetName = "" + rd.URL = "" + rd.MetadataHash = basics.Address{} + rd.Manager = basics.Address{} + rd.Reserve = basics.Address{} + rd.Freeze = basics.Address{} + rd.Clawback = basics.Address{} + hadHolding := (rd.ResourceFlags & ResourceFlagsNotHolding) == ResourceFlagsHolding + rd.ResourceFlags -= rd.ResourceFlags & ResourceFlagsOwnership + rd.ResourceFlags &= ^ResourceFlagsEmptyAsset + if rd.IsEmptyAssetFields() && hadHolding { + rd.ResourceFlags |= ResourceFlagsEmptyAsset + } +} + +// SetAssetParams setter for asset params. +func (rd *ResourcesData) SetAssetParams(ap basics.AssetParams, haveHoldings bool) { + rd.Total = ap.Total + rd.Decimals = ap.Decimals + rd.DefaultFrozen = ap.DefaultFrozen + rd.UnitName = ap.UnitName + rd.AssetName = ap.AssetName + rd.URL = ap.URL + rd.MetadataHash = ap.MetadataHash + rd.Manager = ap.Manager + rd.Reserve = ap.Reserve + rd.Freeze = ap.Freeze + rd.Clawback = ap.Clawback + rd.ResourceFlags |= ResourceFlagsOwnership + if !haveHoldings { + rd.ResourceFlags |= ResourceFlagsNotHolding + } + rd.ResourceFlags &= ^ResourceFlagsEmptyAsset + if rd.IsEmptyAssetFields() { + rd.ResourceFlags |= ResourceFlagsEmptyAsset + } +} + +// GetAssetParams getter for asset params. +func (rd *ResourcesData) GetAssetParams() basics.AssetParams { + ap := basics.AssetParams{ + Total: rd.Total, + Decimals: rd.Decimals, + DefaultFrozen: rd.DefaultFrozen, + UnitName: rd.UnitName, + AssetName: rd.AssetName, + URL: rd.URL, + MetadataHash: rd.MetadataHash, + Manager: rd.Manager, + Reserve: rd.Reserve, + Freeze: rd.Freeze, + Clawback: rd.Clawback, + } + return ap +} + +// ClearAssetHolding clears asset holding. +func (rd *ResourcesData) ClearAssetHolding() { + rd.Amount = 0 + rd.Frozen = false + + rd.ResourceFlags |= ResourceFlagsNotHolding + hadParams := (rd.ResourceFlags & ResourceFlagsOwnership) == ResourceFlagsOwnership + if hadParams && rd.IsEmptyAssetFields() { + rd.ResourceFlags |= ResourceFlagsEmptyAsset + } else { + rd.ResourceFlags &= ^ResourceFlagsEmptyAsset + } +} + +// SetAssetHolding setter for asset holding. +func (rd *ResourcesData) SetAssetHolding(ah basics.AssetHolding) { + rd.Amount = ah.Amount + rd.Frozen = ah.Frozen + rd.ResourceFlags &= ^(ResourceFlagsNotHolding + ResourceFlagsEmptyAsset) + // ResourceFlagsHolding is set implicitly since it is zero + if rd.IsEmptyAssetFields() { + rd.ResourceFlags |= ResourceFlagsEmptyAsset + } +} + +// GetAssetHolding getter for asset holding. +func (rd *ResourcesData) GetAssetHolding() basics.AssetHolding { + return basics.AssetHolding{ + Amount: rd.Amount, + Frozen: rd.Frozen, + } +} + +// ClearAppLocalState clears app local state. +func (rd *ResourcesData) ClearAppLocalState() { + rd.SchemaNumUint = 0 + rd.SchemaNumByteSlice = 0 + rd.KeyValue = nil + + rd.ResourceFlags |= ResourceFlagsNotHolding + hadParams := (rd.ResourceFlags & ResourceFlagsOwnership) == ResourceFlagsOwnership + if hadParams && rd.IsEmptyAppFields() { + rd.ResourceFlags |= ResourceFlagsEmptyApp + } else { + rd.ResourceFlags &= ^ResourceFlagsEmptyApp + } +} + +// SetAppLocalState setter for app local state. +func (rd *ResourcesData) SetAppLocalState(als basics.AppLocalState) { + rd.SchemaNumUint = als.Schema.NumUint + rd.SchemaNumByteSlice = als.Schema.NumByteSlice + rd.KeyValue = als.KeyValue + rd.ResourceFlags &= ^(ResourceFlagsEmptyApp + ResourceFlagsNotHolding) + if rd.IsEmptyAppFields() { + rd.ResourceFlags |= ResourceFlagsEmptyApp + } +} + +// GetAppLocalState getter for app local state. +func (rd *ResourcesData) GetAppLocalState() basics.AppLocalState { + return basics.AppLocalState{ + Schema: basics.StateSchema{ + NumUint: rd.SchemaNumUint, + NumByteSlice: rd.SchemaNumByteSlice, + }, + KeyValue: rd.KeyValue, + } +} + +// ClearAppParams clears the app params. +func (rd *ResourcesData) ClearAppParams() { + rd.ApprovalProgram = nil + rd.ClearStateProgram = nil + rd.GlobalState = nil + rd.LocalStateSchemaNumUint = 0 + rd.LocalStateSchemaNumByteSlice = 0 + rd.GlobalStateSchemaNumUint = 0 + rd.GlobalStateSchemaNumByteSlice = 0 + rd.ExtraProgramPages = 0 + hadHolding := (rd.ResourceFlags & ResourceFlagsNotHolding) == ResourceFlagsHolding + rd.ResourceFlags -= rd.ResourceFlags & ResourceFlagsOwnership + rd.ResourceFlags &= ^ResourceFlagsEmptyApp + if rd.IsEmptyAppFields() && hadHolding { + rd.ResourceFlags |= ResourceFlagsEmptyApp + } +} + +// SetAppParams setter for app params. +func (rd *ResourcesData) SetAppParams(ap basics.AppParams, haveHoldings bool) { + rd.ApprovalProgram = ap.ApprovalProgram + rd.ClearStateProgram = ap.ClearStateProgram + rd.GlobalState = ap.GlobalState + rd.LocalStateSchemaNumUint = ap.LocalStateSchema.NumUint + rd.LocalStateSchemaNumByteSlice = ap.LocalStateSchema.NumByteSlice + rd.GlobalStateSchemaNumUint = ap.GlobalStateSchema.NumUint + rd.GlobalStateSchemaNumByteSlice = ap.GlobalStateSchema.NumByteSlice + rd.ExtraProgramPages = ap.ExtraProgramPages + rd.ResourceFlags |= ResourceFlagsOwnership + if !haveHoldings { + rd.ResourceFlags |= ResourceFlagsNotHolding + } + rd.ResourceFlags &= ^ResourceFlagsEmptyApp + if rd.IsEmptyAppFields() { + rd.ResourceFlags |= ResourceFlagsEmptyApp + } +} + +// GetAppParams getter for app params. +func (rd *ResourcesData) GetAppParams() basics.AppParams { + return basics.AppParams{ + ApprovalProgram: rd.ApprovalProgram, + ClearStateProgram: rd.ClearStateProgram, + GlobalState: rd.GlobalState, + StateSchemas: basics.StateSchemas{ + LocalStateSchema: basics.StateSchema{ + NumUint: rd.LocalStateSchemaNumUint, + NumByteSlice: rd.LocalStateSchemaNumByteSlice, + }, + GlobalStateSchema: basics.StateSchema{ + NumUint: rd.GlobalStateSchemaNumUint, + NumByteSlice: rd.GlobalStateSchemaNumByteSlice, + }, + }, + ExtraProgramPages: rd.ExtraProgramPages, + } +} + +// SetAssetData setter for asset data. +func (rd *ResourcesData) SetAssetData(ap ledgercore.AssetParamsDelta, ah ledgercore.AssetHoldingDelta) { + if ah.Holding != nil { + rd.SetAssetHolding(*ah.Holding) + } else if ah.Deleted { + rd.ClearAssetHolding() + } + if ap.Params != nil { + rd.SetAssetParams(*ap.Params, rd.IsHolding()) + } else if ap.Deleted { + rd.ClearAssetParams() + } +} + +// SetAppData setter for app data. +func (rd *ResourcesData) SetAppData(ap ledgercore.AppParamsDelta, al ledgercore.AppLocalStateDelta) { + if al.LocalState != nil { + rd.SetAppLocalState(*al.LocalState) + } else if al.Deleted { + rd.ClearAppLocalState() + } + if ap.Params != nil { + rd.SetAppParams(*ap.Params, rd.IsHolding()) + } else if ap.Deleted { + rd.ClearAppParams() + } +} + +// Before compares the round numbers of two persistedAccountData and determines if the current persistedAccountData +// happened before the other. +func (pac *PersistedAccountData) Before(other *PersistedAccountData) bool { + return pac.Round < other.Round +} + +// Before compares the round numbers of two persistedResourcesData and determines if the current persistedResourcesData +// happened before the other. +func (prd *PersistedResourcesData) Before(other *PersistedResourcesData) bool { + return prd.Round < other.Round +} + +// Before compares the round numbers of two persistedKVData and determines if the current persistedKVData +// happened before the other. +func (prd PersistedKVData) Before(other *PersistedKVData) bool { + return prd.Round < other.Round +} + +// Before compares the round numbers of two persistedAccountData and determines if the current persistedAccountData +// happened before the other. +func (pac *PersistedOnlineAccountData) Before(other *PersistedOnlineAccountData) bool { + return pac.UpdRound < other.UpdRound +} diff --git a/ledger/store/data_test.go b/ledger/store/data_test.go new file mode 100644 index 0000000000..fdea1e02be --- /dev/null +++ b/ledger/store/data_test.go @@ -0,0 +1,1254 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package store + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + ledgertesting "github.com/algorand/go-algorand/ledger/testing" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" +) + +func TestResourcesDataApp(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + a := require.New(t) + + rd := ResourcesData{} + a.False(rd.IsApp()) + a.True(rd.IsEmpty()) + + rd = MakeResourcesData(1) + a.False(rd.IsApp()) + a.False(rd.IsHolding()) + a.False(rd.IsOwning()) + a.True(rd.IsEmpty()) + + // check empty + appParamsEmpty := basics.AppParams{} + rd = ResourcesData{} + rd.SetAppParams(appParamsEmpty, false) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParamsEmpty, rd.GetAppParams()) + + appLocalEmpty := basics.AppLocalState{} + rd = ResourcesData{} + rd.SetAppLocalState(appLocalEmpty) + a.True(rd.IsApp()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + // check both empty + rd = ResourcesData{} + rd.SetAppLocalState(appLocalEmpty) + rd.SetAppParams(appParamsEmpty, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParamsEmpty, rd.GetAppParams()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + // Since some steps use randomly generated input, the test is run N times + // to cover a larger search space of inputs. + for i := 0; i < 1000; i++ { + // check empty states + non-empty params + appParams := ledgertesting.RandomAppParams() + rd = ResourcesData{} + rd.SetAppLocalState(appLocalEmpty) + rd.SetAppParams(appParams, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParams, rd.GetAppParams()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + appState := ledgertesting.RandomAppLocalState() + rd.SetAppLocalState(appState) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParams, rd.GetAppParams()) + a.Equal(appState, rd.GetAppLocalState()) + + // check ClearAppLocalState + rd.ClearAppLocalState() + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParams, rd.GetAppParams()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + // check ClearAppParams + rd.SetAppLocalState(appState) + rd.ClearAppParams() + a.True(rd.IsApp()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + if appState.Schema.NumEntries() == 0 { + a.True(rd.IsEmptyAppFields()) + } else { + a.False(rd.IsEmptyAppFields()) + } + a.False(rd.IsEmpty()) + a.Equal(appParamsEmpty, rd.GetAppParams()) + a.Equal(appState, rd.GetAppLocalState()) + + // check both clear + rd.ClearAppLocalState() + a.False(rd.IsApp()) + a.False(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.True(rd.IsEmpty()) + a.Equal(appParamsEmpty, rd.GetAppParams()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + // check params clear when non-empty params and empty holding + rd = ResourcesData{} + rd.SetAppLocalState(appLocalEmpty) + rd.SetAppParams(appParams, true) + rd.ClearAppParams() + a.True(rd.IsApp()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + a.Equal(appParamsEmpty, rd.GetAppParams()) + a.Equal(appLocalEmpty, rd.GetAppLocalState()) + + rd = ResourcesData{} + rd.SetAppLocalState(appLocalEmpty) + a.True(rd.IsEmptyAppFields()) + a.True(rd.IsApp()) + a.False(rd.IsEmpty()) + a.Equal(rd.ResourceFlags, ResourceFlagsEmptyApp) + rd.ClearAppLocalState() + a.False(rd.IsApp()) + a.True(rd.IsEmptyAppFields()) + a.True(rd.IsEmpty()) + a.Equal(rd.ResourceFlags, ResourceFlagsNotHolding) + + // check migration flow (accountDataResources) + // 1. both exist and empty + rd = MakeResourcesData(0) + rd.SetAppLocalState(appLocalEmpty) + rd.SetAppParams(appParamsEmpty, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + + // 2. both exist and not empty + rd = MakeResourcesData(0) + rd.SetAppLocalState(appState) + rd.SetAppParams(appParams, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + + // 3. both exist: holding not empty, param is empty + rd = MakeResourcesData(0) + rd.SetAppLocalState(appState) + rd.SetAppParams(appParamsEmpty, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + if appState.Schema.NumEntries() == 0 { + a.True(rd.IsEmptyAppFields()) + } else { + a.False(rd.IsEmptyAppFields()) + } + a.False(rd.IsEmpty()) + + // 4. both exist: holding empty, param is not empty + rd = MakeResourcesData(0) + rd.SetAppLocalState(appLocalEmpty) + rd.SetAppParams(appParams, true) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + + // 5. holding does not exist and params is empty + rd = MakeResourcesData(0) + rd.SetAppParams(appParamsEmpty, false) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + + // 6. holding does not exist and params is not empty + rd = MakeResourcesData(0) + rd.SetAppParams(appParams, false) + a.True(rd.IsApp()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.False(rd.IsEmptyAppFields()) + a.False(rd.IsEmpty()) + + // 7. holding exist and not empty and params does not exist + rd = MakeResourcesData(0) + rd.SetAppLocalState(appState) + a.True(rd.IsApp()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + if appState.Schema.NumEntries() == 0 { + a.True(rd.IsEmptyAppFields()) + } else { + a.False(rd.IsEmptyAppFields()) + } + a.False(rd.IsEmpty()) + + // 8. both do not exist + rd = MakeResourcesData(0) + a.False(rd.IsApp()) + a.False(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAppFields()) + a.True(rd.IsEmpty()) + } +} + +func TestResourcesDataAsset(t *testing.T) { + partitiontest.PartitionTest(t) + + a := require.New(t) + + rd := ResourcesData{} + a.False(rd.IsAsset()) + a.True(rd.IsEmpty()) + + rd = MakeResourcesData(1) + a.False(rd.IsAsset()) + a.False(rd.IsHolding()) + a.False(rd.IsOwning()) + a.True(rd.IsEmpty()) + + // check empty + assetParamsEmpty := basics.AssetParams{} + rd = ResourcesData{} + rd.SetAssetParams(assetParamsEmpty, false) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParamsEmpty, rd.GetAssetParams()) + + assetHoldingEmpty := basics.AssetHolding{} + rd = ResourcesData{} + rd.SetAssetHolding(assetHoldingEmpty) + a.True(rd.IsAsset()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + // check both empty + rd = ResourcesData{} + rd.SetAssetHolding(assetHoldingEmpty) + rd.SetAssetParams(assetParamsEmpty, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParamsEmpty, rd.GetAssetParams()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + // check empty states + non-empty params + assetParams := ledgertesting.RandomAssetParams() + rd = ResourcesData{} + rd.SetAssetHolding(assetHoldingEmpty) + rd.SetAssetParams(assetParams, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParams, rd.GetAssetParams()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + assetHolding := ledgertesting.RandomAssetHolding(true) + rd.SetAssetHolding(assetHolding) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParams, rd.GetAssetParams()) + a.Equal(assetHolding, rd.GetAssetHolding()) + + // check ClearAssetHolding + rd.ClearAssetHolding() + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParams, rd.GetAssetParams()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + // check ClearAssetParams + rd.SetAssetHolding(assetHolding) + rd.ClearAssetParams() + a.True(rd.IsAsset()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParamsEmpty, rd.GetAssetParams()) + a.Equal(assetHolding, rd.GetAssetHolding()) + + // check both clear + rd.ClearAssetHolding() + a.False(rd.IsAsset()) + a.False(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.True(rd.IsEmpty()) + a.Equal(assetParamsEmpty, rd.GetAssetParams()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + // check params clear when non-empty params and empty holding + rd = ResourcesData{} + rd.SetAssetHolding(assetHoldingEmpty) + rd.SetAssetParams(assetParams, true) + rd.ClearAssetParams() + a.True(rd.IsAsset()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + a.Equal(assetParamsEmpty, rd.GetAssetParams()) + a.Equal(assetHoldingEmpty, rd.GetAssetHolding()) + + rd = ResourcesData{} + rd.SetAssetHolding(assetHoldingEmpty) + a.True(rd.IsEmptyAssetFields()) + a.True(rd.IsAsset()) + a.False(rd.IsEmpty()) + a.Equal(rd.ResourceFlags, ResourceFlagsEmptyAsset) + rd.ClearAssetHolding() + a.False(rd.IsAsset()) + a.True(rd.IsEmptyAssetFields()) + a.True(rd.IsEmpty()) + a.Equal(rd.ResourceFlags, ResourceFlagsNotHolding) + + // check migration operations (accountDataResources) + // 1. both exist and empty + rd = MakeResourcesData(0) + rd.SetAssetHolding(assetHoldingEmpty) + rd.SetAssetParams(assetParamsEmpty, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 2. both exist and not empty + rd = MakeResourcesData(0) + rd.SetAssetHolding(assetHolding) + rd.SetAssetParams(assetParams, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 3. both exist: holding not empty, param is empty + rd = MakeResourcesData(0) + rd.SetAssetHolding(assetHolding) + rd.SetAssetParams(assetParamsEmpty, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 4. both exist: holding empty, param is not empty + rd = MakeResourcesData(0) + rd.SetAssetHolding(assetHoldingEmpty) + rd.SetAssetParams(assetParams, true) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 5. holding does not exist and params is empty + rd = MakeResourcesData(0) + rd.SetAssetParams(assetParamsEmpty, false) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 6. holding does not exist and params is not empty + rd = MakeResourcesData(0) + rd.SetAssetParams(assetParams, false) + a.True(rd.IsAsset()) + a.True(rd.IsOwning()) + a.False(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 7. holding exist and not empty and params does not exist + rd = MakeResourcesData(0) + rd.SetAssetHolding(assetHolding) + a.True(rd.IsAsset()) + a.False(rd.IsOwning()) + a.True(rd.IsHolding()) + a.False(rd.IsEmptyAssetFields()) + a.False(rd.IsEmpty()) + + // 8. both do not exist + rd = MakeResourcesData(0) + a.False(rd.IsAsset()) + a.False(rd.IsOwning()) + a.False(rd.IsHolding()) + a.True(rd.IsEmptyAssetFields()) + a.True(rd.IsEmpty()) +} + +// TestResourcesDataSetData checks combinations of old/new values when +// updating resourceData from resourceDelta +func TestResourcesDataSetData(t *testing.T) { + partitiontest.PartitionTest(t) + + a := require.New(t) + + type deltaCode int + const ( + tri deltaCode = iota + 1 + del + emp + act + ) + + // apply deltas encoded as deltaCode to a base ResourcesData for both apps and assets + apply := func(t *testing.T, base ResourcesData, testType basics.CreatableType, pcode, hcode deltaCode) ResourcesData { + if testType == basics.AssetCreatable { + var p ledgercore.AssetParamsDelta + var h ledgercore.AssetHoldingDelta + switch pcode { + case tri: + break + case del: + p = ledgercore.AssetParamsDelta{Deleted: true} + case emp: + p = ledgercore.AssetParamsDelta{Params: &basics.AssetParams{}} + case act: + p = ledgercore.AssetParamsDelta{Params: &basics.AssetParams{Total: 1000}} + default: + t.Logf("invalid pcode: %d", pcode) + t.Fail() + } + switch hcode { + case tri: + break + case del: + h = ledgercore.AssetHoldingDelta{Deleted: true} + case emp: + h = ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{}} + case act: + h = ledgercore.AssetHoldingDelta{Holding: &basics.AssetHolding{Amount: 555}} + default: + t.Logf("invalid hcode: %d", hcode) + t.Fail() + } + base.SetAssetData(p, h) + } else { + var p ledgercore.AppParamsDelta + var h ledgercore.AppLocalStateDelta + switch pcode { + case tri: + break + case del: + p = ledgercore.AppParamsDelta{Deleted: true} + case emp: + p = ledgercore.AppParamsDelta{Params: &basics.AppParams{}} + case act: + p = ledgercore.AppParamsDelta{Params: &basics.AppParams{ClearStateProgram: []byte{4, 5, 6}}} + default: + t.Logf("invalid pcode: %d", pcode) + t.Fail() + } + switch hcode { + case tri: + break + case del: + h = ledgercore.AppLocalStateDelta{Deleted: true} + case emp: + h = ledgercore.AppLocalStateDelta{LocalState: &basics.AppLocalState{}} + case act: + h = ledgercore.AppLocalStateDelta{LocalState: &basics.AppLocalState{Schema: basics.StateSchema{NumByteSlice: 5}}} + default: + t.Logf("invalid hcode: %d", hcode) + t.Fail() + } + base.SetAppData(p, h) + } + + return base + } + + itb := func(i int) (b bool) { + return i != 0 + } + + type testcase struct { + p deltaCode + h deltaCode + isAsset int + isOwning int + isHolding int + isEmptyFields int + isEmpty int + } + + empty := func(testType basics.CreatableType) ResourcesData { + return MakeResourcesData(0) + } + emptyParamsNoHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetParams(basics.AssetParams{}, false) + } else { + rd.SetAppParams(basics.AppParams{}, false) + } + return rd + } + emptyParamsEmptyHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{}) + rd.SetAssetParams(basics.AssetParams{}, true) + } else { + rd.SetAppLocalState(basics.AppLocalState{}) + rd.SetAppParams(basics.AppParams{}, true) + } + return rd + } + emptyParamsNotEmptyHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) + rd.SetAssetParams(basics.AssetParams{}, true) + } else { + rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) + rd.SetAppParams(basics.AppParams{}, true) + } + return rd + } + paramsNoHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetParams(basics.AssetParams{Total: 222}, false) + } else { + rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, false) + } + return rd + } + paramsEmptyHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{}) + rd.SetAssetParams(basics.AssetParams{Total: 222}, true) + } else { + rd.SetAppLocalState(basics.AppLocalState{}) + rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, true) + } + return rd + } + paramsAndHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) + rd.SetAssetParams(basics.AssetParams{Total: 222}, true) + } else { + rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) + rd.SetAppParams(basics.AppParams{ApprovalProgram: []byte{1, 2, 3}}, true) + } + return rd + } + noParamsEmptyHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{}) + } else { + rd.SetAppLocalState(basics.AppLocalState{}) + } + return rd + } + noParamsNotEmptyHolding := func(testType basics.CreatableType) ResourcesData { + rd := MakeResourcesData(0) + if testType == basics.AssetCreatable { + rd.SetAssetHolding(basics.AssetHolding{Amount: 111}) + } else { + rd.SetAppLocalState(basics.AppLocalState{Schema: basics.StateSchema{NumUint: 10}}) + } + return rd + } + + var tests = []struct { + name string + baseRD func(testType basics.CreatableType) ResourcesData + testcases []testcase + }{ + { + "empty_base", empty, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 0, 0, 0, 1, 1}, + {del, tri, 0, 0, 0, 1, 1}, + {emp, tri, 1, 1, 0, 1, 0}, + {act, tri, 1, 1, 0, 0, 0}, + + {tri, del, 0, 0, 0, 1, 1}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 0, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 0, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + + { + "empty_params_no_holding", emptyParamsNoHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 0, 1, 0}, + {del, tri, 0, 0, 0, 1, 1}, + {emp, tri, 1, 1, 0, 1, 0}, + {act, tri, 1, 1, 0, 0, 0}, + + {tri, del, 1, 1, 0, 1, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "empty_params_empty_holding", emptyParamsEmptyHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 1, 1, 0}, + {del, tri, 1, 0, 1, 1, 0}, + {emp, tri, 1, 1, 1, 1, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 1, 1, 0, 1, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "empty_params_not_empty_holding", emptyParamsNotEmptyHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 1, 0, 0}, + {del, tri, 1, 0, 1, 0, 0}, + {emp, tri, 1, 1, 1, 0, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 1, 1, 0, 1, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "params_no_holding", paramsNoHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 0, 0, 0}, + {del, tri, 0, 0, 0, 1, 1}, + {emp, tri, 1, 1, 0, 1, 0}, + {act, tri, 1, 1, 0, 0, 0}, + + {tri, del, 1, 1, 0, 0, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 0, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "params_empty_holding", paramsEmptyHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 1, 0, 0}, + {del, tri, 1, 0, 1, 1, 0}, + {emp, tri, 1, 1, 1, 1, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 1, 1, 0, 0, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 0, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "params_and_holding", paramsAndHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 1, 1, 0, 0}, + {del, tri, 1, 0, 1, 0, 0}, + {emp, tri, 1, 1, 1, 0, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 1, 1, 0, 0, 0}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 1, 1, 0, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 1, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "no_params_empty_holding", noParamsEmptyHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 0, 1, 1, 0}, + {del, tri, 1, 0, 1, 1, 0}, + {emp, tri, 1, 1, 1, 1, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 0, 0, 0, 1, 1}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 0, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 0, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + { + "no_params_not_empty_holding", noParamsNotEmptyHolding, + []testcase{ + // IsAsset, IsOwning, IsHolding, IsEmptyAssetFields, IsEmpty + {tri, tri, 1, 0, 1, 0, 0}, + {del, tri, 1, 0, 1, 0, 0}, + {emp, tri, 1, 1, 1, 0, 0}, + {act, tri, 1, 1, 1, 0, 0}, + + {tri, del, 0, 0, 0, 1, 1}, + {del, del, 0, 0, 0, 1, 1}, + {emp, del, 1, 1, 0, 1, 0}, + {act, del, 1, 1, 0, 0, 0}, + + {tri, emp, 1, 0, 1, 1, 0}, + {del, emp, 1, 0, 1, 1, 0}, + {emp, emp, 1, 1, 1, 1, 0}, + {act, emp, 1, 1, 1, 0, 0}, + + {tri, act, 1, 0, 1, 0, 0}, + {del, act, 1, 0, 1, 0, 0}, + {emp, act, 1, 1, 1, 0, 0}, + {act, act, 1, 1, 1, 0, 0}, + }, + }, + } + for _, testType := range []basics.CreatableType{basics.AssetCreatable, basics.AppCreatable} { + for _, test := range tests { + var testTypeStr string + if testType == basics.AssetCreatable { + testTypeStr = "asset" + } else { + testTypeStr = "app" + } + t.Run(fmt.Sprintf("test_%s_%s", testTypeStr, test.name), func(t *testing.T) { + for i, ts := range test.testcases { + t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { + rd := test.baseRD(testType) + rd = apply(t, rd, testType, ts.p, ts.h) + if testType == basics.AssetCreatable { + a.Equal(itb(ts.isAsset), rd.IsAsset()) + a.Equal(itb(ts.isEmptyFields), rd.IsEmptyAssetFields()) + a.False(rd.IsApp()) + a.True(rd.IsEmptyAppFields()) + } else { + a.Equal(itb(ts.isAsset), rd.IsApp()) + a.Equal(itb(ts.isEmptyFields), rd.IsEmptyAppFields()) + a.False(rd.IsAsset()) + a.True(rd.IsEmptyAssetFields()) + } + a.Equal(itb(ts.isOwning), rd.IsOwning()) + a.Equal(itb(ts.isHolding), rd.IsHolding()) + a.Equal(itb(ts.isEmpty), rd.IsEmpty()) + }) + } + }) + } + } +} + +// TestResourceDataRoundtripConversion ensures that basics.AppLocalState, basics.AppParams, +// basics.AssetHolding, and basics.AssetParams can be converted to resourcesData and back without +// losing any data. It uses reflection to be sure that no new fields are omitted. +// +// In other words, this test makes sure any new fields in basics.AppLocalState, basics.AppParams, +// basics.AssetHolding, or basics.AssetParam also get added to resourcesData. +func TestResourceDataRoundtripConversion(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + t.Run("basics.AppLocalState", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&basics.AppLocalState{}) + basicsAppLocalState := *randObj.(*basics.AppLocalState) + + var data ResourcesData + data.SetAppLocalState(basicsAppLocalState) + roundTripAppLocalState := data.GetAppLocalState() + + require.Equal(t, basicsAppLocalState, roundTripAppLocalState) + } + }) + + t.Run("basics.AppParams", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&basics.AppParams{}) + basicsAppParams := *randObj.(*basics.AppParams) + + for _, haveHoldings := range []bool{true, false} { + var data ResourcesData + data.SetAppParams(basicsAppParams, haveHoldings) + roundTripAppParams := data.GetAppParams() + + require.Equal(t, basicsAppParams, roundTripAppParams) + } + } + }) + + t.Run("basics.AssetHolding", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&basics.AssetHolding{}) + basicsAssetHolding := *randObj.(*basics.AssetHolding) + + var data ResourcesData + data.SetAssetHolding(basicsAssetHolding) + roundTripAssetHolding := data.GetAssetHolding() + + require.Equal(t, basicsAssetHolding, roundTripAssetHolding) + } + }) + + t.Run("basics.AssetParams", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&basics.AssetParams{}) + basicsAssetParams := *randObj.(*basics.AssetParams) + + for _, haveHoldings := range []bool{true, false} { + var data ResourcesData + data.SetAssetParams(basicsAssetParams, haveHoldings) + roundTripAssetParams := data.GetAssetParams() + + require.Equal(t, basicsAssetParams, roundTripAssetParams) + } + } + }) +} + +// TestBaseAccountDataRoundtripConversion ensures that baseAccountData can be converted to +// ledgercore.AccountData and basics.AccountData and back without losing any data. It uses +// reflection to be sure that no new fields are omitted. +// +// In other words, this test makes sure any new fields in baseAccountData also get added to +// ledgercore.AccountData and basics.AccountData. You should add a manual override in this test if +// the field really only belongs in baseAccountData. +func TestBaseAccountDataRoundtripConversion(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + t.Run("ledgercore.AccountData", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&BaseAccountData{}) + baseAccount := *randObj.(*BaseAccountData) + + ledgercoreAccount := baseAccount.GetLedgerCoreAccountData() + var roundTripAccount BaseAccountData + roundTripAccount.SetCoreAccountData(&ledgercoreAccount) + + // Manually set UpdateRound, since it is lost in GetLedgerCoreAccountData + roundTripAccount.UpdateRound = baseAccount.UpdateRound + + require.Equal(t, baseAccount, roundTripAccount) + } + }) + + t.Run("basics.AccountData", func(t *testing.T) { + t.Parallel() + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&BaseAccountData{}) + baseAccount := *randObj.(*BaseAccountData) + + basicsAccount := baseAccount.GetAccountData() + var roundTripAccount BaseAccountData + roundTripAccount.SetAccountData(&basicsAccount) + + // Manually set UpdateRound, since it is lost in GetAccountData + roundTripAccount.UpdateRound = baseAccount.UpdateRound + + // Manually set resources, since resource information is lost in GetAccountData + roundTripAccount.TotalAssetParams = baseAccount.TotalAssetParams + roundTripAccount.TotalAssets = baseAccount.TotalAssets + roundTripAccount.TotalAppLocalStates = baseAccount.TotalAppLocalStates + roundTripAccount.TotalAppParams = baseAccount.TotalAppParams + + require.Equal(t, baseAccount, roundTripAccount) + } + }) +} + +// TestBasicsAccountDataRoundtripConversion ensures that basics.AccountData can be converted to +// baseAccountData and back without losing any data. It uses reflection to be sure that this test is +// always up-to-date with new fields. +// +// In other words, this test makes sure any new fields in basics.AccountData also get added to +// baseAccountData. +func TestBasicsAccountDataRoundtripConversion(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&basics.AccountData{}) + basicsAccount := *randObj.(*basics.AccountData) + + var baseAccount BaseAccountData + baseAccount.SetAccountData(&basicsAccount) + roundTripAccount := baseAccount.GetAccountData() + + // Manually set resources, since GetAccountData doesn't attempt to restore them + roundTripAccount.AssetParams = basicsAccount.AssetParams + roundTripAccount.Assets = basicsAccount.Assets + roundTripAccount.AppLocalStates = basicsAccount.AppLocalStates + roundTripAccount.AppParams = basicsAccount.AppParams + + require.Equal(t, basicsAccount, roundTripAccount) + require.Equal(t, uint64(len(roundTripAccount.AssetParams)), baseAccount.TotalAssetParams) + require.Equal(t, uint64(len(roundTripAccount.Assets)), baseAccount.TotalAssets) + require.Equal(t, uint64(len(roundTripAccount.AppLocalStates)), baseAccount.TotalAppLocalStates) + require.Equal(t, uint64(len(roundTripAccount.AppParams)), baseAccount.TotalAppParams) + } +} + +// TestLedgercoreAccountDataRoundtripConversion ensures that ledgercore.AccountData can be converted +// to baseAccountData and back without losing any data. It uses reflection to be sure that no new +// fields are omitted. +// +// In other words, this test makes sure any new fields in ledgercore.AccountData also get added to +// baseAccountData. +func TestLedgercoreAccountDataRoundtripConversion(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + for i := 0; i < 1000; i++ { + randObj, _ := protocol.RandomizeObject(&ledgercore.AccountData{}) + ledgercoreAccount := *randObj.(*ledgercore.AccountData) + + var baseAccount BaseAccountData + baseAccount.SetCoreAccountData(&ledgercoreAccount) + roundTripAccount := baseAccount.GetLedgerCoreAccountData() + + require.Equal(t, ledgercoreAccount, roundTripAccount) + } +} + +func TestBaseAccountDataIsEmpty(t *testing.T) { + partitiontest.PartitionTest(t) + positiveTesting := func(t *testing.T) { + var ba BaseAccountData + require.True(t, ba.IsEmpty()) + for i := 0; i < 20; i++ { + h := crypto.Hash([]byte{byte(i)}) + rnd := binary.BigEndian.Uint64(h[:]) + ba.UpdateRound = rnd + require.True(t, ba.IsEmpty()) + } + } + var empty BaseAccountData + negativeTesting := func(t *testing.T) { + for i := 0; i < 10000; i++ { + randObj, _ := protocol.RandomizeObjectField(&BaseAccountData{}) + ba := randObj.(*BaseAccountData) + if *ba == empty || ba.UpdateRound != 0 { + continue + } + require.False(t, ba.IsEmpty(), "base account : %v", ba) + } + } + structureTesting := func(t *testing.T) { + encoding, err := json.Marshal(&empty) + zeros32 := "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0" + expectedEncoding := `{"Status":0,"MicroAlgos":{"Raw":0},"RewardsBase":0,"RewardedMicroAlgos":{"Raw":0},"AuthAddr":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ","TotalAppSchemaNumUint":0,"TotalAppSchemaNumByteSlice":0,"TotalExtraAppPages":0,"TotalAssetParams":0,"TotalAssets":0,"TotalAppParams":0,"TotalAppLocalStates":0,"TotalBoxes":0,"TotalBoxBytes":0,"VoteID":[` + zeros32 + `],"SelectionID":[` + zeros32 + `],"VoteFirstValid":0,"VoteLastValid":0,"VoteKeyDilution":0,"StateProofID":[` + zeros32 + `,` + zeros32 + `],"UpdateRound":0}` + require.NoError(t, err) + require.Equal(t, expectedEncoding, string(encoding)) + } + t.Run("Positive", positiveTesting) + t.Run("Negative", negativeTesting) + t.Run("Structure", structureTesting) + +} + +func TestBaseOnlineAccountDataIsEmpty(t *testing.T) { + partitiontest.PartitionTest(t) + + positiveTesting := func(t *testing.T) { + var ba BaseOnlineAccountData + require.True(t, ba.IsEmpty()) + require.True(t, ba.IsVotingEmpty()) + ba.MicroAlgos.Raw = 100 + require.True(t, ba.IsVotingEmpty()) + ba.RewardsBase = 200 + require.True(t, ba.IsVotingEmpty()) + } + var empty BaseOnlineAccountData + negativeTesting := func(t *testing.T) { + for i := 0; i < 10; i++ { + randObj, _ := protocol.RandomizeObjectField(&BaseOnlineAccountData{}) + ba := randObj.(*BaseOnlineAccountData) + if *ba == empty { + continue + } + require.False(t, ba.IsEmpty(), "base account : %v", ba) + break + } + { + var ba BaseOnlineAccountData + ba.MicroAlgos.Raw = 100 + require.False(t, ba.IsEmpty()) + } + { + var ba BaseOnlineAccountData + ba.RewardsBase = 200 + require.False(t, ba.IsEmpty()) + } + } + structureTesting := func(t *testing.T) { + encoding, err := json.Marshal(&empty) + zeros32 := "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0" + expectedEncoding := `{"VoteID":[` + zeros32 + `],"SelectionID":[` + zeros32 + `],"VoteFirstValid":0,"VoteLastValid":0,"VoteKeyDilution":0,"StateProofID":[` + zeros32 + `,` + zeros32 + `],"MicroAlgos":{"Raw":0},"RewardsBase":0}` + require.NoError(t, err) + require.Equal(t, expectedEncoding, string(encoding)) + } + t.Run("Positive", positiveTesting) + t.Run("Negative", negativeTesting) + t.Run("Structure", structureTesting) + +} + +func TestBaseOnlineAccountDataGettersSetters(t *testing.T) { + partitiontest.PartitionTest(t) + + proto := config.Consensus[protocol.ConsensusCurrentVersion] + addr := ledgertesting.RandomAddress() + data := ledgertesting.RandomAccountData(1) + data.Status = basics.Online + crypto.RandBytes(data.VoteID[:]) + crypto.RandBytes(data.SelectionID[:]) + crypto.RandBytes(data.StateProofID[:]) + data.VoteFirstValid = basics.Round(crypto.RandUint64()) + data.VoteLastValid = basics.Round(crypto.RandUint64()) // int64 is the max sqlite can store + data.VoteKeyDilution = crypto.RandUint64() + + var ba BaseOnlineAccountData + ad := ledgercore.ToAccountData(data) + ba.SetCoreAccountData(&ad) + + require.Equal(t, data.MicroAlgos, ba.MicroAlgos) + require.Equal(t, data.RewardsBase, ba.RewardsBase) + require.Equal(t, data.VoteID, ba.VoteID) + require.Equal(t, data.SelectionID, ba.SelectionID) + require.Equal(t, data.VoteFirstValid, ba.VoteFirstValid) + require.Equal(t, data.VoteLastValid, ba.VoteLastValid) + require.Equal(t, data.VoteKeyDilution, ba.VoteKeyDilution) + require.Equal(t, data.StateProofID, ba.StateProofID) + + normBalance := basics.NormalizedOnlineAccountBalance(basics.Online, data.RewardsBase, data.MicroAlgos, proto) + require.Equal(t, normBalance, ba.NormalizedOnlineBalance(proto)) + oa := ba.GetOnlineAccount(addr, normBalance) + + require.Equal(t, addr, oa.Address) + require.Equal(t, ba.MicroAlgos, oa.MicroAlgos) + require.Equal(t, ba.RewardsBase, oa.RewardsBase) + require.Equal(t, normBalance, oa.NormalizedOnlineBalance) + require.Equal(t, ba.VoteFirstValid, oa.VoteFirstValid) + require.Equal(t, ba.VoteLastValid, oa.VoteLastValid) + require.Equal(t, ba.StateProofID, oa.StateProofID) + + rewardsLevel := uint64(1) + microAlgos, _, _ := basics.WithUpdatedRewards( + proto, basics.Online, oa.MicroAlgos, basics.MicroAlgos{}, ba.RewardsBase, rewardsLevel, + ) + oad := ba.GetOnlineAccountData(proto, rewardsLevel) + + require.Equal(t, microAlgos, oad.MicroAlgosWithRewards) + require.Equal(t, ba.VoteID, oad.VoteID) + require.Equal(t, ba.SelectionID, oad.SelectionID) + require.Equal(t, ba.StateProofID, oad.StateProofID) + require.Equal(t, ba.VoteFirstValid, oad.VoteFirstValid) + require.Equal(t, ba.VoteLastValid, oad.VoteLastValid) + require.Equal(t, ba.VoteKeyDilution, oad.VoteKeyDilution) +} + +func TestBaseVotingDataGettersSetters(t *testing.T) { + partitiontest.PartitionTest(t) + + data := ledgertesting.RandomAccountData(1) + data.Status = basics.Online + crypto.RandBytes(data.VoteID[:]) + crypto.RandBytes(data.SelectionID[:]) + crypto.RandBytes(data.StateProofID[:]) + data.VoteFirstValid = basics.Round(crypto.RandUint64()) + data.VoteLastValid = basics.Round(crypto.RandUint64()) // int64 is the max sqlite can store + data.VoteKeyDilution = crypto.RandUint64() + + var bv BaseVotingData + require.True(t, bv.IsEmpty()) + + ad := ledgercore.ToAccountData(data) + bv.SetCoreAccountData(&ad) + + require.False(t, bv.IsEmpty()) + require.Equal(t, data.VoteID, bv.VoteID) + require.Equal(t, data.SelectionID, bv.SelectionID) + require.Equal(t, data.VoteFirstValid, bv.VoteFirstValid) + require.Equal(t, data.VoteLastValid, bv.VoteLastValid) + require.Equal(t, data.VoteKeyDilution, bv.VoteKeyDilution) + require.Equal(t, data.StateProofID, bv.StateProofID) +} + +func TestBaseOnlineAccountDataReflect(t *testing.T) { + partitiontest.PartitionTest(t) + + require.Equal(t, 4, reflect.TypeOf(BaseOnlineAccountData{}).NumField(), "update all getters and setters for baseOnlineAccountData and change the field count") +} + +func TestBaseVotingDataReflect(t *testing.T) { + partitiontest.PartitionTest(t) + + require.Equal(t, 7, reflect.TypeOf(BaseVotingData{}).NumField(), "update all getters and setters for baseVotingData and change the field count") +} diff --git a/ledger/store/interface.go b/ledger/store/interface.go new file mode 100644 index 0000000000..835330042d --- /dev/null +++ b/ledger/store/interface.go @@ -0,0 +1,75 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package store + +import "github.com/algorand/go-algorand/data/basics" + +// AccountsWriter is the write interface for: +// - accounts, resources, app kvs, creatables +type AccountsWriter interface { + InsertAccount(addr basics.Address, normBalance uint64, data BaseAccountData) (rowid int64, err error) + DeleteAccount(rowid int64) (rowsAffected int64, err error) + UpdateAccount(rowid int64, normBalance uint64, data BaseAccountData) (rowsAffected int64, err error) + + InsertResource(addrid int64, aidx basics.CreatableIndex, data ResourcesData) (rowid int64, err error) + DeleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) + UpdateResource(addrid int64, aidx basics.CreatableIndex, data ResourcesData) (rowsAffected int64, err error) + + UpsertKvPair(key string, value []byte) error + DeleteKvPair(key string) error + + InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) + DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) + + Close() +} + +// AccountsReader is the read interface for: +// - accounts, resources, app kvs, creatables +type AccountsReader interface { + ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) + + LookupAccount(addr basics.Address) (data PersistedAccountData, err error) + + LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data PersistedResourcesData, err error) + LookupAllResources(addr basics.Address) (data []PersistedResourcesData, rnd basics.Round, err error) + + LookupKeyValue(key string) (pv PersistedKVData, err error) + LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) + + LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) + + Close() +} + +// OnlineAccountsWriter is the write interface for: +// - online accounts +type OnlineAccountsWriter interface { + InsertOnlineAccount(addr basics.Address, normBalance uint64, data BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) + + Close() +} + +// OnlineAccountsReader is the read interface for: +// - online accounts +type OnlineAccountsReader interface { + LookupOnline(addr basics.Address, rnd basics.Round) (data PersistedOnlineAccountData, err error) + LookupOnlineTotalsHistory(round basics.Round) (basics.MicroAlgos, error) + LookupOnlineHistory(addr basics.Address) (result []PersistedOnlineAccountData, rnd basics.Round, err error) + + Close() +} diff --git a/ledger/store/msgp_gen.go b/ledger/store/msgp_gen.go new file mode 100644 index 0000000000..0fcd1fb9d2 --- /dev/null +++ b/ledger/store/msgp_gen.go @@ -0,0 +1,1880 @@ +package store + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "github.com/algorand/msgp/msgp" + + "github.com/algorand/go-algorand/config" +) + +// The following msgp objects are implemented in this file: +// BaseAccountData +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// BaseOnlineAccountData +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// BaseVotingData +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// ResourceFlags +// |-----> MarshalMsg +// |-----> CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> Msgsize +// |-----> MsgIsZero +// +// ResourcesData +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// + +// MarshalMsg implements msgp.Marshaler +func (z *BaseAccountData) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(21) + var zb0001Mask uint32 /* 23 bits */ + if (*z).BaseVotingData.VoteID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x1 + } + if (*z).BaseVotingData.SelectionID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).BaseVotingData.VoteFirstValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).BaseVotingData.VoteLastValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).BaseVotingData.VoteKeyDilution == 0 { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).BaseVotingData.StateProofID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x20 + } + if (*z).Status.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x100 + } + if (*z).MicroAlgos.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x200 + } + if (*z).RewardsBase == 0 { + zb0001Len-- + zb0001Mask |= 0x400 + } + if (*z).RewardedMicroAlgos.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x800 + } + if (*z).AuthAddr.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x1000 + } + if (*z).TotalAppSchemaNumUint == 0 { + zb0001Len-- + zb0001Mask |= 0x2000 + } + if (*z).TotalAppSchemaNumByteSlice == 0 { + zb0001Len-- + zb0001Mask |= 0x4000 + } + if (*z).TotalExtraAppPages == 0 { + zb0001Len-- + zb0001Mask |= 0x8000 + } + if (*z).TotalAssetParams == 0 { + zb0001Len-- + zb0001Mask |= 0x10000 + } + if (*z).TotalAssets == 0 { + zb0001Len-- + zb0001Mask |= 0x20000 + } + if (*z).TotalAppParams == 0 { + zb0001Len-- + zb0001Mask |= 0x40000 + } + if (*z).TotalAppLocalStates == 0 { + zb0001Len-- + zb0001Mask |= 0x80000 + } + if (*z).TotalBoxes == 0 { + zb0001Len-- + zb0001Mask |= 0x100000 + } + if (*z).TotalBoxBytes == 0 { + zb0001Len-- + zb0001Mask |= 0x200000 + } + if (*z).UpdateRound == 0 { + zb0001Len-- + zb0001Mask |= 0x400000 + } + // variable map header, size zb0001Len + o = msgp.AppendMapHeader(o, zb0001Len) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "A" + o = append(o, 0xa1, 0x41) + o = (*z).BaseVotingData.VoteID.MarshalMsg(o) + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "B" + o = append(o, 0xa1, 0x42) + o = (*z).BaseVotingData.SelectionID.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "C" + o = append(o, 0xa1, 0x43) + o = (*z).BaseVotingData.VoteFirstValid.MarshalMsg(o) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "D" + o = append(o, 0xa1, 0x44) + o = (*z).BaseVotingData.VoteLastValid.MarshalMsg(o) + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "E" + o = append(o, 0xa1, 0x45) + o = msgp.AppendUint64(o, (*z).BaseVotingData.VoteKeyDilution) + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "F" + o = append(o, 0xa1, 0x46) + o = (*z).BaseVotingData.StateProofID.MarshalMsg(o) + } + if (zb0001Mask & 0x100) == 0 { // if not empty + // string "a" + o = append(o, 0xa1, 0x61) + o = (*z).Status.MarshalMsg(o) + } + if (zb0001Mask & 0x200) == 0 { // if not empty + // string "b" + o = append(o, 0xa1, 0x62) + o = (*z).MicroAlgos.MarshalMsg(o) + } + if (zb0001Mask & 0x400) == 0 { // if not empty + // string "c" + o = append(o, 0xa1, 0x63) + o = msgp.AppendUint64(o, (*z).RewardsBase) + } + if (zb0001Mask & 0x800) == 0 { // if not empty + // string "d" + o = append(o, 0xa1, 0x64) + o = (*z).RewardedMicroAlgos.MarshalMsg(o) + } + if (zb0001Mask & 0x1000) == 0 { // if not empty + // string "e" + o = append(o, 0xa1, 0x65) + o = (*z).AuthAddr.MarshalMsg(o) + } + if (zb0001Mask & 0x2000) == 0 { // if not empty + // string "f" + o = append(o, 0xa1, 0x66) + o = msgp.AppendUint64(o, (*z).TotalAppSchemaNumUint) + } + if (zb0001Mask & 0x4000) == 0 { // if not empty + // string "g" + o = append(o, 0xa1, 0x67) + o = msgp.AppendUint64(o, (*z).TotalAppSchemaNumByteSlice) + } + if (zb0001Mask & 0x8000) == 0 { // if not empty + // string "h" + o = append(o, 0xa1, 0x68) + o = msgp.AppendUint32(o, (*z).TotalExtraAppPages) + } + if (zb0001Mask & 0x10000) == 0 { // if not empty + // string "i" + o = append(o, 0xa1, 0x69) + o = msgp.AppendUint64(o, (*z).TotalAssetParams) + } + if (zb0001Mask & 0x20000) == 0 { // if not empty + // string "j" + o = append(o, 0xa1, 0x6a) + o = msgp.AppendUint64(o, (*z).TotalAssets) + } + if (zb0001Mask & 0x40000) == 0 { // if not empty + // string "k" + o = append(o, 0xa1, 0x6b) + o = msgp.AppendUint64(o, (*z).TotalAppParams) + } + if (zb0001Mask & 0x80000) == 0 { // if not empty + // string "l" + o = append(o, 0xa1, 0x6c) + o = msgp.AppendUint64(o, (*z).TotalAppLocalStates) + } + if (zb0001Mask & 0x100000) == 0 { // if not empty + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendUint64(o, (*z).TotalBoxes) + } + if (zb0001Mask & 0x200000) == 0 { // if not empty + // string "n" + o = append(o, 0xa1, 0x6e) + o = msgp.AppendUint64(o, (*z).TotalBoxBytes) + } + if (zb0001Mask & 0x400000) == 0 { // if not empty + // string "z" + o = append(o, 0xa1, 0x7a) + o = msgp.AppendUint64(o, (*z).UpdateRound) + } + } + return +} + +func (_ *BaseAccountData) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseAccountData) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BaseAccountData) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Status.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Status") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "MicroAlgos") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "RewardsBase") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).RewardedMicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "RewardedMicroAlgos") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).AuthAddr.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "AuthAddr") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAppSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAppSchemaNumUint") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAppSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAppSchemaNumByteSlice") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalExtraAppPages, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalExtraAppPages") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAssetParams, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAssetParams") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAssets, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAssets") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAppParams, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAppParams") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAppLocalStates, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAppLocalStates") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalBoxes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalBoxes") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalBoxBytes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalBoxBytes") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SelectionID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).BaseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProofID") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "UpdateRound") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = BaseAccountData{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "a": + bts, err = (*z).Status.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + case "b": + bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "MicroAlgos") + return + } + case "c": + (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RewardsBase") + return + } + case "d": + bts, err = (*z).RewardedMicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "RewardedMicroAlgos") + return + } + case "e": + bts, err = (*z).AuthAddr.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "AuthAddr") + return + } + case "f": + (*z).TotalAppSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAppSchemaNumUint") + return + } + case "g": + (*z).TotalAppSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAppSchemaNumByteSlice") + return + } + case "h": + (*z).TotalExtraAppPages, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalExtraAppPages") + return + } + case "i": + (*z).TotalAssetParams, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAssetParams") + return + } + case "j": + (*z).TotalAssets, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAssets") + return + } + case "k": + (*z).TotalAppParams, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAppParams") + return + } + case "l": + (*z).TotalAppLocalStates, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAppLocalStates") + return + } + case "m": + (*z).TotalBoxes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalBoxes") + return + } + case "n": + (*z).TotalBoxBytes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalBoxBytes") + return + } + case "A": + bts, err = (*z).BaseVotingData.VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteID") + return + } + case "B": + bts, err = (*z).BaseVotingData.SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SelectionID") + return + } + case "C": + bts, err = (*z).BaseVotingData.VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteFirstValid") + return + } + case "D": + bts, err = (*z).BaseVotingData.VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteLastValid") + return + } + case "E": + (*z).BaseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "VoteKeyDilution") + return + } + case "F": + bts, err = (*z).BaseVotingData.StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProofID") + return + } + case "z": + (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateRound") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *BaseAccountData) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseAccountData) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BaseAccountData) Msgsize() (s int) { + s = 3 + 2 + (*z).Status.Msgsize() + 2 + (*z).MicroAlgos.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).RewardedMicroAlgos.Msgsize() + 2 + (*z).AuthAddr.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + (*z).BaseVotingData.VoteID.Msgsize() + 2 + (*z).BaseVotingData.SelectionID.Msgsize() + 2 + (*z).BaseVotingData.VoteFirstValid.Msgsize() + 2 + (*z).BaseVotingData.VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).BaseVotingData.StateProofID.Msgsize() + 2 + msgp.Uint64Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z *BaseAccountData) MsgIsZero() bool { + return ((*z).Status.MsgIsZero()) && ((*z).MicroAlgos.MsgIsZero()) && ((*z).RewardsBase == 0) && ((*z).RewardedMicroAlgos.MsgIsZero()) && ((*z).AuthAddr.MsgIsZero()) && ((*z).TotalAppSchemaNumUint == 0) && ((*z).TotalAppSchemaNumByteSlice == 0) && ((*z).TotalExtraAppPages == 0) && ((*z).TotalAssetParams == 0) && ((*z).TotalAssets == 0) && ((*z).TotalAppParams == 0) && ((*z).TotalAppLocalStates == 0) && ((*z).TotalBoxes == 0) && ((*z).TotalBoxBytes == 0) && ((*z).BaseVotingData.VoteID.MsgIsZero()) && ((*z).BaseVotingData.SelectionID.MsgIsZero()) && ((*z).BaseVotingData.VoteFirstValid.MsgIsZero()) && ((*z).BaseVotingData.VoteLastValid.MsgIsZero()) && ((*z).BaseVotingData.VoteKeyDilution == 0) && ((*z).BaseVotingData.StateProofID.MsgIsZero()) && ((*z).UpdateRound == 0) +} + +// MarshalMsg implements msgp.Marshaler +func (z *BaseOnlineAccountData) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(8) + var zb0001Mask uint16 /* 10 bits */ + if (*z).BaseVotingData.VoteID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x1 + } + if (*z).BaseVotingData.SelectionID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).BaseVotingData.VoteFirstValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).BaseVotingData.VoteLastValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).BaseVotingData.VoteKeyDilution == 0 { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).BaseVotingData.StateProofID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x20 + } + if (*z).MicroAlgos.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40 + } + if (*z).RewardsBase == 0 { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "A" + o = append(o, 0xa1, 0x41) + o = (*z).BaseVotingData.VoteID.MarshalMsg(o) + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "B" + o = append(o, 0xa1, 0x42) + o = (*z).BaseVotingData.SelectionID.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "C" + o = append(o, 0xa1, 0x43) + o = (*z).BaseVotingData.VoteFirstValid.MarshalMsg(o) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "D" + o = append(o, 0xa1, 0x44) + o = (*z).BaseVotingData.VoteLastValid.MarshalMsg(o) + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "E" + o = append(o, 0xa1, 0x45) + o = msgp.AppendUint64(o, (*z).BaseVotingData.VoteKeyDilution) + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "F" + o = append(o, 0xa1, 0x46) + o = (*z).BaseVotingData.StateProofID.MarshalMsg(o) + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "Y" + o = append(o, 0xa1, 0x59) + o = (*z).MicroAlgos.MarshalMsg(o) + } + if (zb0001Mask & 0x80) == 0 { // if not empty + // string "Z" + o = append(o, 0xa1, 0x5a) + o = msgp.AppendUint64(o, (*z).RewardsBase) + } + } + return +} + +func (_ *BaseOnlineAccountData) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseOnlineAccountData) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BaseOnlineAccountData) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SelectionID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).BaseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BaseVotingData.StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProofID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "MicroAlgos") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "RewardsBase") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = BaseOnlineAccountData{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "A": + bts, err = (*z).BaseVotingData.VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteID") + return + } + case "B": + bts, err = (*z).BaseVotingData.SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SelectionID") + return + } + case "C": + bts, err = (*z).BaseVotingData.VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteFirstValid") + return + } + case "D": + bts, err = (*z).BaseVotingData.VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteLastValid") + return + } + case "E": + (*z).BaseVotingData.VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "VoteKeyDilution") + return + } + case "F": + bts, err = (*z).BaseVotingData.StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProofID") + return + } + case "Y": + bts, err = (*z).MicroAlgos.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "MicroAlgos") + return + } + case "Z": + (*z).RewardsBase, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RewardsBase") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *BaseOnlineAccountData) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseOnlineAccountData) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BaseOnlineAccountData) Msgsize() (s int) { + s = 1 + 2 + (*z).BaseVotingData.VoteID.Msgsize() + 2 + (*z).BaseVotingData.SelectionID.Msgsize() + 2 + (*z).BaseVotingData.VoteFirstValid.Msgsize() + 2 + (*z).BaseVotingData.VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).BaseVotingData.StateProofID.Msgsize() + 2 + (*z).MicroAlgos.Msgsize() + 2 + msgp.Uint64Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z *BaseOnlineAccountData) MsgIsZero() bool { + return ((*z).BaseVotingData.VoteID.MsgIsZero()) && ((*z).BaseVotingData.SelectionID.MsgIsZero()) && ((*z).BaseVotingData.VoteFirstValid.MsgIsZero()) && ((*z).BaseVotingData.VoteLastValid.MsgIsZero()) && ((*z).BaseVotingData.VoteKeyDilution == 0) && ((*z).BaseVotingData.StateProofID.MsgIsZero()) && ((*z).MicroAlgos.MsgIsZero()) && ((*z).RewardsBase == 0) +} + +// MarshalMsg implements msgp.Marshaler +func (z *BaseVotingData) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(6) + var zb0001Mask uint8 /* 7 bits */ + if (*z).VoteID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x1 + } + if (*z).SelectionID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).VoteFirstValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).VoteLastValid.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).VoteKeyDilution == 0 { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).StateProofID.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x20 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "A" + o = append(o, 0xa1, 0x41) + o = (*z).VoteID.MarshalMsg(o) + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "B" + o = append(o, 0xa1, 0x42) + o = (*z).SelectionID.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "C" + o = append(o, 0xa1, 0x43) + o = (*z).VoteFirstValid.MarshalMsg(o) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "D" + o = append(o, 0xa1, 0x44) + o = (*z).VoteLastValid.MarshalMsg(o) + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "E" + o = append(o, 0xa1, 0x45) + o = msgp.AppendUint64(o, (*z).VoteKeyDilution) + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "F" + o = append(o, 0xa1, 0x46) + o = (*z).StateProofID.MarshalMsg(o) + } + } + return +} + +func (_ *BaseVotingData) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseVotingData) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BaseVotingData) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SelectionID") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteFirstValid") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteLastValid") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "VoteKeyDilution") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProofID") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = BaseVotingData{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "A": + bts, err = (*z).VoteID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteID") + return + } + case "B": + bts, err = (*z).SelectionID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SelectionID") + return + } + case "C": + bts, err = (*z).VoteFirstValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteFirstValid") + return + } + case "D": + bts, err = (*z).VoteLastValid.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VoteLastValid") + return + } + case "E": + (*z).VoteKeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "VoteKeyDilution") + return + } + case "F": + bts, err = (*z).StateProofID.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProofID") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *BaseVotingData) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*BaseVotingData) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BaseVotingData) Msgsize() (s int) { + s = 1 + 2 + (*z).VoteID.Msgsize() + 2 + (*z).SelectionID.Msgsize() + 2 + (*z).VoteFirstValid.Msgsize() + 2 + (*z).VoteLastValid.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).StateProofID.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *BaseVotingData) MsgIsZero() bool { + return ((*z).VoteID.MsgIsZero()) && ((*z).SelectionID.MsgIsZero()) && ((*z).VoteFirstValid.MsgIsZero()) && ((*z).VoteLastValid.MsgIsZero()) && ((*z).VoteKeyDilution == 0) && ((*z).StateProofID.MsgIsZero()) +} + +// MarshalMsg implements msgp.Marshaler +func (z ResourceFlags) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +func (_ ResourceFlags) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(ResourceFlags) + if !ok { + _, ok = (z).(*ResourceFlags) + } + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResourceFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ResourceFlags(zb0001) + } + o = bts + return +} + +func (_ *ResourceFlags) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*ResourceFlags) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ResourceFlags) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z ResourceFlags) MsgIsZero() bool { + return z == 0 +} + +// MarshalMsg implements msgp.Marshaler +func (z *ResourcesData) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0002Len := uint32(26) + var zb0002Mask uint32 /* 27 bits */ + if (*z).Total == 0 { + zb0002Len-- + zb0002Mask |= 0x2 + } + if (*z).Decimals == 0 { + zb0002Len-- + zb0002Mask |= 0x4 + } + if (*z).DefaultFrozen == false { + zb0002Len-- + zb0002Mask |= 0x8 + } + if (*z).UnitName == "" { + zb0002Len-- + zb0002Mask |= 0x10 + } + if (*z).AssetName == "" { + zb0002Len-- + zb0002Mask |= 0x20 + } + if (*z).URL == "" { + zb0002Len-- + zb0002Mask |= 0x40 + } + if (*z).MetadataHash == ([32]byte{}) { + zb0002Len-- + zb0002Mask |= 0x80 + } + if (*z).Manager.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x100 + } + if (*z).Reserve.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x200 + } + if (*z).Freeze.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x400 + } + if (*z).Clawback.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x800 + } + if (*z).Amount == 0 { + zb0002Len-- + zb0002Mask |= 0x1000 + } + if (*z).Frozen == false { + zb0002Len-- + zb0002Mask |= 0x2000 + } + if (*z).SchemaNumUint == 0 { + zb0002Len-- + zb0002Mask |= 0x4000 + } + if (*z).SchemaNumByteSlice == 0 { + zb0002Len-- + zb0002Mask |= 0x8000 + } + if (*z).KeyValue.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x10000 + } + if len((*z).ApprovalProgram) == 0 { + zb0002Len-- + zb0002Mask |= 0x20000 + } + if len((*z).ClearStateProgram) == 0 { + zb0002Len-- + zb0002Mask |= 0x40000 + } + if (*z).GlobalState.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x80000 + } + if (*z).LocalStateSchemaNumUint == 0 { + zb0002Len-- + zb0002Mask |= 0x100000 + } + if (*z).LocalStateSchemaNumByteSlice == 0 { + zb0002Len-- + zb0002Mask |= 0x200000 + } + if (*z).GlobalStateSchemaNumUint == 0 { + zb0002Len-- + zb0002Mask |= 0x400000 + } + if (*z).GlobalStateSchemaNumByteSlice == 0 { + zb0002Len-- + zb0002Mask |= 0x800000 + } + if (*z).ExtraProgramPages == 0 { + zb0002Len-- + zb0002Mask |= 0x1000000 + } + if (*z).ResourceFlags == 0 { + zb0002Len-- + zb0002Mask |= 0x2000000 + } + if (*z).UpdateRound == 0 { + zb0002Len-- + zb0002Mask |= 0x4000000 + } + // variable map header, size zb0002Len + o = msgp.AppendMapHeader(o, zb0002Len) + if zb0002Len != 0 { + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "a" + o = append(o, 0xa1, 0x61) + o = msgp.AppendUint64(o, (*z).Total) + } + if (zb0002Mask & 0x4) == 0 { // if not empty + // string "b" + o = append(o, 0xa1, 0x62) + o = msgp.AppendUint32(o, (*z).Decimals) + } + if (zb0002Mask & 0x8) == 0 { // if not empty + // string "c" + o = append(o, 0xa1, 0x63) + o = msgp.AppendBool(o, (*z).DefaultFrozen) + } + if (zb0002Mask & 0x10) == 0 { // if not empty + // string "d" + o = append(o, 0xa1, 0x64) + o = msgp.AppendString(o, (*z).UnitName) + } + if (zb0002Mask & 0x20) == 0 { // if not empty + // string "e" + o = append(o, 0xa1, 0x65) + o = msgp.AppendString(o, (*z).AssetName) + } + if (zb0002Mask & 0x40) == 0 { // if not empty + // string "f" + o = append(o, 0xa1, 0x66) + o = msgp.AppendString(o, (*z).URL) + } + if (zb0002Mask & 0x80) == 0 { // if not empty + // string "g" + o = append(o, 0xa1, 0x67) + o = msgp.AppendBytes(o, ((*z).MetadataHash)[:]) + } + if (zb0002Mask & 0x100) == 0 { // if not empty + // string "h" + o = append(o, 0xa1, 0x68) + o = (*z).Manager.MarshalMsg(o) + } + if (zb0002Mask & 0x200) == 0 { // if not empty + // string "i" + o = append(o, 0xa1, 0x69) + o = (*z).Reserve.MarshalMsg(o) + } + if (zb0002Mask & 0x400) == 0 { // if not empty + // string "j" + o = append(o, 0xa1, 0x6a) + o = (*z).Freeze.MarshalMsg(o) + } + if (zb0002Mask & 0x800) == 0 { // if not empty + // string "k" + o = append(o, 0xa1, 0x6b) + o = (*z).Clawback.MarshalMsg(o) + } + if (zb0002Mask & 0x1000) == 0 { // if not empty + // string "l" + o = append(o, 0xa1, 0x6c) + o = msgp.AppendUint64(o, (*z).Amount) + } + if (zb0002Mask & 0x2000) == 0 { // if not empty + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendBool(o, (*z).Frozen) + } + if (zb0002Mask & 0x4000) == 0 { // if not empty + // string "n" + o = append(o, 0xa1, 0x6e) + o = msgp.AppendUint64(o, (*z).SchemaNumUint) + } + if (zb0002Mask & 0x8000) == 0 { // if not empty + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendUint64(o, (*z).SchemaNumByteSlice) + } + if (zb0002Mask & 0x10000) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + o = (*z).KeyValue.MarshalMsg(o) + } + if (zb0002Mask & 0x20000) == 0 { // if not empty + // string "q" + o = append(o, 0xa1, 0x71) + o = msgp.AppendBytes(o, (*z).ApprovalProgram) + } + if (zb0002Mask & 0x40000) == 0 { // if not empty + // string "r" + o = append(o, 0xa1, 0x72) + o = msgp.AppendBytes(o, (*z).ClearStateProgram) + } + if (zb0002Mask & 0x80000) == 0 { // if not empty + // string "s" + o = append(o, 0xa1, 0x73) + o = (*z).GlobalState.MarshalMsg(o) + } + if (zb0002Mask & 0x100000) == 0 { // if not empty + // string "t" + o = append(o, 0xa1, 0x74) + o = msgp.AppendUint64(o, (*z).LocalStateSchemaNumUint) + } + if (zb0002Mask & 0x200000) == 0 { // if not empty + // string "u" + o = append(o, 0xa1, 0x75) + o = msgp.AppendUint64(o, (*z).LocalStateSchemaNumByteSlice) + } + if (zb0002Mask & 0x400000) == 0 { // if not empty + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendUint64(o, (*z).GlobalStateSchemaNumUint) + } + if (zb0002Mask & 0x800000) == 0 { // if not empty + // string "w" + o = append(o, 0xa1, 0x77) + o = msgp.AppendUint64(o, (*z).GlobalStateSchemaNumByteSlice) + } + if (zb0002Mask & 0x1000000) == 0 { // if not empty + // string "x" + o = append(o, 0xa1, 0x78) + o = msgp.AppendUint32(o, (*z).ExtraProgramPages) + } + if (zb0002Mask & 0x2000000) == 0 { // if not empty + // string "y" + o = append(o, 0xa1, 0x79) + o = msgp.AppendUint8(o, uint8((*z).ResourceFlags)) + } + if (zb0002Mask & 0x4000000) == 0 { // if not empty + // string "z" + o = append(o, 0xa1, 0x7a) + o = msgp.AppendUint64(o, (*z).UpdateRound) + } + } + return +} + +func (_ *ResourcesData) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*ResourcesData) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResourcesData) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 > 0 { + zb0002-- + (*z).Total, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Total") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).Decimals, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Decimals") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).DefaultFrozen, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "DefaultFrozen") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).UnitName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "UnitName") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).AssetName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "AssetName") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).URL, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "URL") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = msgp.ReadExactBytes(bts, ((*z).MetadataHash)[:]) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "MetadataHash") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).Manager.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Manager") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).Reserve.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Reserve") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).Freeze.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Freeze") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).Clawback.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Clawback") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).Amount, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Amount") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).Frozen, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Frozen") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).SchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SchemaNumUint") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).SchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SchemaNumByteSlice") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).KeyValue.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KeyValue") + return + } + } + if zb0002 > 0 { + zb0002-- + var zb0004 int + zb0004, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ApprovalProgram") + return + } + if zb0004 > config.MaxAvailableAppProgramLen { + err = msgp.ErrOverflow(uint64(zb0004), uint64(config.MaxAvailableAppProgramLen)) + return + } + (*z).ApprovalProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ApprovalProgram) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ApprovalProgram") + return + } + } + if zb0002 > 0 { + zb0002-- + var zb0005 int + zb0005, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ClearStateProgram") + return + } + if zb0005 > config.MaxAvailableAppProgramLen { + err = msgp.ErrOverflow(uint64(zb0005), uint64(config.MaxAvailableAppProgramLen)) + return + } + (*z).ClearStateProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ClearStateProgram) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ClearStateProgram") + return + } + } + if zb0002 > 0 { + zb0002-- + bts, err = (*z).GlobalState.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "GlobalState") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).LocalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "LocalStateSchemaNumUint") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).LocalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "LocalStateSchemaNumByteSlice") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).GlobalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "GlobalStateSchemaNumUint") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).GlobalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "GlobalStateSchemaNumByteSlice") + return + } + } + if zb0002 > 0 { + zb0002-- + (*z).ExtraProgramPages, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ExtraProgramPages") + return + } + } + if zb0002 > 0 { + zb0002-- + { + var zb0006 uint8 + zb0006, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ResourceFlags") + return + } + (*z).ResourceFlags = ResourceFlags(zb0006) + } + } + if zb0002 > 0 { + zb0002-- + (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "UpdateRound") + return + } + } + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = ResourcesData{} + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "a": + (*z).Total, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "b": + (*z).Decimals, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Decimals") + return + } + case "c": + (*z).DefaultFrozen, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DefaultFrozen") + return + } + case "d": + (*z).UnitName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UnitName") + return + } + case "e": + (*z).AssetName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AssetName") + return + } + case "f": + (*z).URL, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "URL") + return + } + case "g": + bts, err = msgp.ReadExactBytes(bts, ((*z).MetadataHash)[:]) + if err != nil { + err = msgp.WrapError(err, "MetadataHash") + return + } + case "h": + bts, err = (*z).Manager.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Manager") + return + } + case "i": + bts, err = (*z).Reserve.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Reserve") + return + } + case "j": + bts, err = (*z).Freeze.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Freeze") + return + } + case "k": + bts, err = (*z).Clawback.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Clawback") + return + } + case "l": + (*z).Amount, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Amount") + return + } + case "m": + (*z).Frozen, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Frozen") + return + } + case "n": + (*z).SchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SchemaNumUint") + return + } + case "o": + (*z).SchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SchemaNumByteSlice") + return + } + case "p": + bts, err = (*z).KeyValue.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "KeyValue") + return + } + case "q": + var zb0007 int + zb0007, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "ApprovalProgram") + return + } + if zb0007 > config.MaxAvailableAppProgramLen { + err = msgp.ErrOverflow(uint64(zb0007), uint64(config.MaxAvailableAppProgramLen)) + return + } + (*z).ApprovalProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ApprovalProgram) + if err != nil { + err = msgp.WrapError(err, "ApprovalProgram") + return + } + case "r": + var zb0008 int + zb0008, err = msgp.ReadBytesBytesHeader(bts) + if err != nil { + err = msgp.WrapError(err, "ClearStateProgram") + return + } + if zb0008 > config.MaxAvailableAppProgramLen { + err = msgp.ErrOverflow(uint64(zb0008), uint64(config.MaxAvailableAppProgramLen)) + return + } + (*z).ClearStateProgram, bts, err = msgp.ReadBytesBytes(bts, (*z).ClearStateProgram) + if err != nil { + err = msgp.WrapError(err, "ClearStateProgram") + return + } + case "s": + bts, err = (*z).GlobalState.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "GlobalState") + return + } + case "t": + (*z).LocalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LocalStateSchemaNumUint") + return + } + case "u": + (*z).LocalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LocalStateSchemaNumByteSlice") + return + } + case "v": + (*z).GlobalStateSchemaNumUint, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GlobalStateSchemaNumUint") + return + } + case "w": + (*z).GlobalStateSchemaNumByteSlice, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GlobalStateSchemaNumByteSlice") + return + } + case "x": + (*z).ExtraProgramPages, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExtraProgramPages") + return + } + case "y": + { + var zb0009 uint8 + zb0009, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResourceFlags") + return + } + (*z).ResourceFlags = ResourceFlags(zb0009) + } + case "z": + (*z).UpdateRound, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateRound") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *ResourcesData) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*ResourcesData) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ResourcesData) Msgsize() (s int) { + s = 3 + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.BoolSize + 2 + msgp.StringPrefixSize + len((*z).UnitName) + 2 + msgp.StringPrefixSize + len((*z).AssetName) + 2 + msgp.StringPrefixSize + len((*z).URL) + 2 + msgp.ArrayHeaderSize + (32 * (msgp.ByteSize)) + 2 + (*z).Manager.Msgsize() + 2 + (*z).Reserve.Msgsize() + 2 + (*z).Freeze.Msgsize() + 2 + (*z).Clawback.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.BoolSize + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + (*z).KeyValue.Msgsize() + 2 + msgp.BytesPrefixSize + len((*z).ApprovalProgram) + 2 + msgp.BytesPrefixSize + len((*z).ClearStateProgram) + 2 + (*z).GlobalState.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint32Size + 2 + msgp.Uint8Size + 2 + msgp.Uint64Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z *ResourcesData) MsgIsZero() bool { + return ((*z).Total == 0) && ((*z).Decimals == 0) && ((*z).DefaultFrozen == false) && ((*z).UnitName == "") && ((*z).AssetName == "") && ((*z).URL == "") && ((*z).MetadataHash == ([32]byte{})) && ((*z).Manager.MsgIsZero()) && ((*z).Reserve.MsgIsZero()) && ((*z).Freeze.MsgIsZero()) && ((*z).Clawback.MsgIsZero()) && ((*z).Amount == 0) && ((*z).Frozen == false) && ((*z).SchemaNumUint == 0) && ((*z).SchemaNumByteSlice == 0) && ((*z).KeyValue.MsgIsZero()) && (len((*z).ApprovalProgram) == 0) && (len((*z).ClearStateProgram) == 0) && ((*z).GlobalState.MsgIsZero()) && ((*z).LocalStateSchemaNumUint == 0) && ((*z).LocalStateSchemaNumByteSlice == 0) && ((*z).GlobalStateSchemaNumUint == 0) && ((*z).GlobalStateSchemaNumByteSlice == 0) && ((*z).ExtraProgramPages == 0) && ((*z).ResourceFlags == 0) && ((*z).UpdateRound == 0) +} diff --git a/ledger/store/msgp_gen_test.go b/ledger/store/msgp_gen_test.go new file mode 100644 index 0000000000..a6d2ec2fb4 --- /dev/null +++ b/ledger/store/msgp_gen_test.go @@ -0,0 +1,255 @@ +//go:build !skip_msgp_testing +// +build !skip_msgp_testing + +package store + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "testing" + + "github.com/algorand/msgp/msgp" + + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestMarshalUnmarshalBaseAccountData(t *testing.T) { + partitiontest.PartitionTest(t) + v := BaseAccountData{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingBaseAccountData(t *testing.T) { + protocol.RunEncodingTest(t, &BaseAccountData{}) +} + +func BenchmarkMarshalMsgBaseAccountData(b *testing.B) { + v := BaseAccountData{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBaseAccountData(b *testing.B) { + v := BaseAccountData{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBaseAccountData(b *testing.B) { + v := BaseAccountData{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBaseOnlineAccountData(t *testing.T) { + partitiontest.PartitionTest(t) + v := BaseOnlineAccountData{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingBaseOnlineAccountData(t *testing.T) { + protocol.RunEncodingTest(t, &BaseOnlineAccountData{}) +} + +func BenchmarkMarshalMsgBaseOnlineAccountData(b *testing.B) { + v := BaseOnlineAccountData{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBaseOnlineAccountData(b *testing.B) { + v := BaseOnlineAccountData{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBaseOnlineAccountData(b *testing.B) { + v := BaseOnlineAccountData{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBaseVotingData(t *testing.T) { + partitiontest.PartitionTest(t) + v := BaseVotingData{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingBaseVotingData(t *testing.T) { + protocol.RunEncodingTest(t, &BaseVotingData{}) +} + +func BenchmarkMarshalMsgBaseVotingData(b *testing.B) { + v := BaseVotingData{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBaseVotingData(b *testing.B) { + v := BaseVotingData{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBaseVotingData(b *testing.B) { + v := BaseVotingData{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResourcesData(t *testing.T) { + partitiontest.PartitionTest(t) + v := ResourcesData{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingResourcesData(t *testing.T) { + protocol.RunEncodingTest(t, &ResourcesData{}) +} + +func BenchmarkMarshalMsgResourcesData(b *testing.B) { + v := ResourcesData{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResourcesData(b *testing.B) { + v := ResourcesData{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResourcesData(b *testing.B) { + v := ResourcesData{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/ledger/store/sql.go b/ledger/store/sql.go new file mode 100644 index 0000000000..ae8166faef --- /dev/null +++ b/ledger/store/sql.go @@ -0,0 +1,716 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package store + +import ( + "database/sql" + "fmt" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +// accountsDbQueries is used to cache a prepared SQL statement to look up +// the state of a single account. +type accountsDbQueries struct { + listCreatablesStmt *sql.Stmt + lookupAccountStmt *sql.Stmt + lookupResourcesStmt *sql.Stmt + lookupAllResourcesStmt *sql.Stmt + lookupKvPairStmt *sql.Stmt + lookupKeysByRangeStmt *sql.Stmt + lookupCreatorStmt *sql.Stmt +} + +type onlineAccountsDbQueries struct { + lookupOnlineStmt *sql.Stmt + lookupOnlineHistoryStmt *sql.Stmt + lookupOnlineTotalsStmt *sql.Stmt +} + +type accountsSQLWriter struct { + insertCreatableIdxStmt, deleteCreatableIdxStmt *sql.Stmt + deleteByRowIDStmt, insertStmt, updateStmt *sql.Stmt + deleteResourceStmt, insertResourceStmt, updateResourceStmt *sql.Stmt + deleteKvPairStmt, upsertKvPairStmt *sql.Stmt +} + +type onlineAccountsSQLWriter struct { + insertStmt, updateStmt *sql.Stmt +} + +// AccountsInitDbQueries constructs an AccountsReader backed by sql queries. +func AccountsInitDbQueries(q db.Queryable) (*accountsDbQueries, error) { + var err error + qs := &accountsDbQueries{} + + qs.listCreatablesStmt, err = q.Prepare("SELECT acctrounds.rnd, assetcreators.asset, assetcreators.creator FROM acctrounds LEFT JOIN assetcreators ON assetcreators.asset <= ? AND assetcreators.ctype = ? WHERE acctrounds.id='acctbase' ORDER BY assetcreators.asset desc LIMIT ?") + if err != nil { + return nil, err + } + + qs.lookupAccountStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, accountbase.data FROM acctrounds LEFT JOIN accountbase ON address=? WHERE id='acctbase'") + if err != nil { + return nil, err + } + + qs.lookupResourcesStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, resources.data FROM acctrounds LEFT JOIN accountbase ON accountbase.address = ? LEFT JOIN resources ON accountbase.rowid = resources.addrid AND resources.aidx = ? WHERE id='acctbase'") + if err != nil { + return nil, err + } + + qs.lookupAllResourcesStmt, err = q.Prepare("SELECT accountbase.rowid, acctrounds.rnd, resources.aidx, resources.data FROM acctrounds LEFT JOIN accountbase ON accountbase.address = ? LEFT JOIN resources ON accountbase.rowid = resources.addrid WHERE id='acctbase'") + if err != nil { + return nil, err + } + + qs.lookupKvPairStmt, err = q.Prepare("SELECT acctrounds.rnd, kvstore.value FROM acctrounds LEFT JOIN kvstore ON key = ? WHERE id='acctbase';") + if err != nil { + return nil, err + } + + qs.lookupKeysByRangeStmt, err = q.Prepare("SELECT acctrounds.rnd, kvstore.key FROM acctrounds LEFT JOIN kvstore ON kvstore.key >= ? AND kvstore.key < ? WHERE id='acctbase'") + if err != nil { + return nil, err + } + + qs.lookupCreatorStmt, err = q.Prepare("SELECT acctrounds.rnd, assetcreators.creator FROM acctrounds LEFT JOIN assetcreators ON asset = ? AND ctype = ? WHERE id='acctbase'") + if err != nil { + return nil, err + } + + return qs, nil +} + +// OnlineAccountsInitDbQueries constructs an OnlineAccountsReader backed by sql queries. +func OnlineAccountsInitDbQueries(r db.Queryable) (*onlineAccountsDbQueries, error) { + var err error + qs := &onlineAccountsDbQueries{} + + qs.lookupOnlineStmt, err = r.Prepare("SELECT onlineaccounts.rowid, onlineaccounts.updround, acctrounds.rnd, onlineaccounts.data FROM acctrounds LEFT JOIN onlineaccounts ON address=? AND updround <= ? WHERE id='acctbase' ORDER BY updround DESC LIMIT 1") + if err != nil { + return nil, err + } + + qs.lookupOnlineHistoryStmt, err = r.Prepare("SELECT onlineaccounts.rowid, onlineaccounts.updround, acctrounds.rnd, onlineaccounts.data FROM acctrounds LEFT JOIN onlineaccounts ON address=? WHERE id='acctbase' ORDER BY updround ASC") + if err != nil { + return nil, err + } + + qs.lookupOnlineTotalsStmt, err = r.Prepare("SELECT data FROM onlineroundparamstail WHERE rnd=?") + if err != nil { + return nil, err + } + return qs, nil +} + +// MakeOnlineAccountsSQLWriter constructs an OnlineAccountsWriter backed by sql queries. +func MakeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) { + w = new(onlineAccountsSQLWriter) + + if hasAccounts { + w.insertStmt, err = tx.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)") + if err != nil { + return + } + + w.updateStmt, err = tx.Prepare("UPDATE onlineaccounts SET normalizedonlinebalance = ?, data = ?, updround = ?, votelastvalid =? WHERE rowid = ?") + if err != nil { + return + } + } + + return +} + +// MakeAccountsSQLWriter constructs an AccountsWriter backed by sql queries. +func MakeAccountsSQLWriter(tx *sql.Tx, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) { + w = new(accountsSQLWriter) + + if hasAccounts { + w.deleteByRowIDStmt, err = tx.Prepare("DELETE FROM accountbase WHERE rowid=?") + if err != nil { + return + } + + w.insertStmt, err = tx.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)") + if err != nil { + return + } + + w.updateStmt, err = tx.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?") + if err != nil { + return + } + } + + if hasResources { + w.deleteResourceStmt, err = tx.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?") + if err != nil { + return + } + + w.insertResourceStmt, err = tx.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") + if err != nil { + return + } + + w.updateResourceStmt, err = tx.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?") + if err != nil { + return + } + } + + if hasKvPairs { + w.upsertKvPairStmt, err = tx.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value") + if err != nil { + return + } + + w.deleteKvPairStmt, err = tx.Prepare("DELETE FROM kvstore WHERE key=?") + if err != nil { + return + } + } + + if hasCreatables { + w.insertCreatableIdxStmt, err = tx.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)") + if err != nil { + return + } + + w.deleteCreatableIdxStmt, err = tx.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?") + if err != nil { + return + } + } + return +} + +// ListCreatables returns an array of CreatableLocator which have CreatableIndex smaller or equal to maxIdx and are of the provided CreatableType. +func (qs *accountsDbQueries) ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) { + err = db.Retry(func() error { + // Query for assets in range + rows, err := qs.listCreatablesStmt.Query(maxIdx, ctype, maxResults) + if err != nil { + return err + } + defer rows.Close() + + // For each row, copy into a new CreatableLocator and append to results + var buf []byte + var cl basics.CreatableLocator + var creatableIndex sql.NullInt64 + for rows.Next() { + err = rows.Scan(&dbRound, &creatableIndex, &buf) + if err != nil { + return err + } + if !creatableIndex.Valid { + // we received an entry without any index. This would happen only on the first entry when there are no creatables of the requested type. + break + } + cl.Index = basics.CreatableIndex(creatableIndex.Int64) + copy(cl.Creator[:], buf) + cl.Type = ctype + results = append(results, cl) + } + return nil + }) + return +} + +// sql.go has the following contradictory comments: + +// Reference types such as []byte are only valid until the next call to Scan +// and should not be retained. Their underlying memory is owned by the driver. +// If retention is necessary, copy their values before the next call to Scan. + +// If a dest argument has type *[]byte, Scan saves in that argument a +// copy of the corresponding data. The copy is owned by the caller and +// can be modified and held indefinitely. The copy can be avoided by +// using an argument of type *RawBytes instead; see the documentation +// for RawBytes for restrictions on its use. + +// After check source code, a []byte slice destination is definitely cloned. + +// LookupKeyValue returns the application boxed value associated with the key. +func (qs *accountsDbQueries) LookupKeyValue(key string) (pv PersistedKVData, err error) { + err = db.Retry(func() error { + var val []byte + // Cast to []byte to avoid interpretation as character string, see note in upsertKvPair + err := qs.lookupKvPairStmt.QueryRow([]byte(key)).Scan(&pv.Round, &val) + if err != nil { + // this should never happen; it indicates that we don't have a current round in the acctrounds table. + if err == sql.ErrNoRows { + // Return the zero value of data + err = fmt.Errorf("unable to query value for key %v : %w", key, err) + } + return err + } + if val != nil { // We got a non-null value, so it exists + pv.Value = val + return nil + } + // we don't have that key, just return pv with the database round (pv.value==nil) + return nil + }) + return +} + +// LookupKeysByPrefix returns a set of application boxed values matching the prefix. +func (qs *accountsDbQueries) LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) { + start, end := keyPrefixIntervalPreprocessing([]byte(prefix)) + if end == nil { + // Not an expected use case, it's asking for all keys, or all keys + // prefixed by some number of 0xFF bytes. + return 0, fmt.Errorf("lookup by strange prefix %#v", prefix) + } + err = db.Retry(func() error { + var rows *sql.Rows + rows, err = qs.lookupKeysByRangeStmt.Query(start, end) + if err != nil { + return err + } + defer rows.Close() + + var v sql.NullString + + for rows.Next() { + if resultCount == maxKeyNum { + return nil + } + err = rows.Scan(&round, &v) + if err != nil { + return err + } + if v.Valid { + if _, ok := results[v.String]; ok { + continue + } + results[v.String] = true + resultCount++ + } + } + return nil + }) + return +} + +// keyPrefixIntervalPreprocessing is implemented to generate an interval for DB queries that look up keys by prefix. +// Such DB query was designed this way, to trigger the binary search optimization in SQLITE3. +// The DB comparison for blob typed primary key is lexicographic, i.e., byte by byte. +// In this way, we can introduce an interval that a primary key should be >= some prefix, < some prefix increment. +// A corner case to consider is that, the prefix has last byte 0xFF, or the prefix is full of 0xFF. +// - The first case can be solved by carrying, e.g., prefix = 0x1EFF -> interval being >= 0x1EFF and < 0x1F +// - The second case can be solved by disregarding the upper limit, i.e., prefix = 0xFFFF -> interval being >= 0xFFFF +// Another corner case to consider is empty byte, []byte{} or nil. +// - In both cases, the results are interval >= "", i.e., returns []byte{} for prefix, and nil for prefixIncr. +func keyPrefixIntervalPreprocessing(prefix []byte) ([]byte, []byte) { + if prefix == nil { + prefix = []byte{} + } + prefixIncr := make([]byte, len(prefix)) + copy(prefixIncr, prefix) + for i := len(prefix) - 1; i >= 0; i-- { + currentByteIncr := int(prefix[i]) + 1 + if currentByteIncr > 0xFF { + prefixIncr = prefixIncr[:len(prefixIncr)-1] + continue + } + prefixIncr[i] = byte(currentByteIncr) + return prefix, prefixIncr + } + return prefix, nil +} + +// LookupCreator returns the address and round of the creator. +func (qs *accountsDbQueries) LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) { + err = db.Retry(func() error { + var buf []byte + err := qs.lookupCreatorStmt.QueryRow(cidx, ctype).Scan(&dbRound, &buf) + + // this shouldn't happen unless we can't figure the round number. + if err == sql.ErrNoRows { + return fmt.Errorf("lookupCreator was unable to retrieve round number") + } + + // Some other database error + if err != nil { + return err + } + + if len(buf) > 0 { + ok = true + copy(addr[:], buf) + } + return nil + }) + return +} + +// LookupResources returns the requested resource. +func (qs *accountsDbQueries) LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data PersistedResourcesData, err error) { + err = db.Retry(func() error { + var buf []byte + var rowid sql.NullInt64 + err := qs.lookupResourcesStmt.QueryRow(addr[:], aidx).Scan(&rowid, &data.Round, &buf) + if err == nil { + data.Aidx = aidx + if len(buf) > 0 && rowid.Valid { + data.Addrid = rowid.Int64 + err = protocol.Decode(buf, &data.Data) + if err != nil { + return err + } + if ctype == basics.AssetCreatable && !data.Data.IsAsset() { + return fmt.Errorf("lookupResources asked for an asset but got %v", data.Data) + } + if ctype == basics.AppCreatable && !data.Data.IsApp() { + return fmt.Errorf("lookupResources asked for an app but got %v", data.Data) + } + return nil + } + data.Data = MakeResourcesData(0) + // we don't have that account, just return the database round. + return nil + } + + // this should never happen; it indicates that we don't have a current round in the acctrounds table. + if err == sql.ErrNoRows { + // Return the zero value of data + return fmt.Errorf("unable to query resource data for address %v aidx %v ctype %v : %w", addr, aidx, ctype, err) + } + return err + }) + return +} + +// LookupAllResources returns all resources associated with the given address. +func (qs *accountsDbQueries) LookupAllResources(addr basics.Address) (data []PersistedResourcesData, rnd basics.Round, err error) { + err = db.Retry(func() error { + // Query for all resources + rows, err := qs.lookupAllResourcesStmt.Query(addr[:]) + if err != nil { + return err + } + defer rows.Close() + + var addrid, aidx sql.NullInt64 + var dbRound basics.Round + data = nil + var buf []byte + for rows.Next() { + err := rows.Scan(&addrid, &dbRound, &aidx, &buf) + if err != nil { + return err + } + if !addrid.Valid || !aidx.Valid { + // we received an entry without any index. This would happen only on the first entry when there are no resources for this address. + // ensure this is the first entry, set the round and return + if len(data) != 0 { + return fmt.Errorf("lookupAllResources: unexpected invalid result on non-first resource record: (%v, %v)", addrid.Valid, aidx.Valid) + } + rnd = dbRound + break + } + var resData ResourcesData + err = protocol.Decode(buf, &resData) + if err != nil { + return err + } + data = append(data, PersistedResourcesData{ + Addrid: addrid.Int64, + Aidx: basics.CreatableIndex(aidx.Int64), + Data: resData, + Round: dbRound, + }) + rnd = dbRound + } + return nil + }) + return +} + +// LookupAccount looks up for a the account data given it's address. It returns the persistedAccountData, which includes the current database round and the matching +// account data, if such was found. If no matching account data could be found for the given address, an empty account data would +// be retrieved. +func (qs *accountsDbQueries) LookupAccount(addr basics.Address) (data PersistedAccountData, err error) { + err = db.Retry(func() error { + var buf []byte + var rowid sql.NullInt64 + err := qs.lookupAccountStmt.QueryRow(addr[:]).Scan(&rowid, &data.Round, &buf) + if err == nil { + data.Addr = addr + if len(buf) > 0 && rowid.Valid { + data.Rowid = rowid.Int64 + err = protocol.Decode(buf, &data.AccountData) + return err + } + // we don't have that account, just return the database round. + return nil + } + + // this should never happen; it indicates that we don't have a current round in the acctrounds table. + if err == sql.ErrNoRows { + // Return the zero value of data + return fmt.Errorf("unable to query account data for address %v : %w", addr, err) + } + + return err + }) + return +} + +// LookupOnline returns the online account data for the given address. +func (qs *onlineAccountsDbQueries) LookupOnline(addr basics.Address, rnd basics.Round) (data PersistedOnlineAccountData, err error) { + err = db.Retry(func() error { + var buf []byte + var rowid sql.NullInt64 + var updround sql.NullInt64 + err := qs.lookupOnlineStmt.QueryRow(addr[:], rnd).Scan(&rowid, &updround, &data.Round, &buf) + if err == nil { + data.Addr = addr + if len(buf) > 0 && rowid.Valid && updround.Valid { + data.Rowid = rowid.Int64 + data.UpdRound = basics.Round(updround.Int64) + err = protocol.Decode(buf, &data.AccountData) + return err + } + // we don't have that account, just return the database round. + return nil + } + + // this should never happen; it indicates that we don't have a current round in the acctrounds table. + if err == sql.ErrNoRows { + // Return the zero value of data + return fmt.Errorf("unable to query online account data for address %v : %w", addr, err) + } + + return err + }) + return +} + +func (qs *onlineAccountsDbQueries) LookupOnlineTotalsHistory(round basics.Round) (basics.MicroAlgos, error) { + data := ledgercore.OnlineRoundParamsData{} + err := db.Retry(func() error { + row := qs.lookupOnlineTotalsStmt.QueryRow(round) + var buf []byte + err := row.Scan(&buf) + if err != nil { + return err + } + err = protocol.Decode(buf, &data) + if err != nil { + return err + } + return nil + }) + return basics.MicroAlgos{Raw: data.OnlineSupply}, err +} + +func (qs *onlineAccountsDbQueries) LookupOnlineHistory(addr basics.Address) (result []PersistedOnlineAccountData, rnd basics.Round, err error) { + err = db.Retry(func() error { + rows, err := qs.lookupOnlineHistoryStmt.Query(addr[:]) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var buf []byte + data := PersistedOnlineAccountData{} + err := rows.Scan(&data.Rowid, &data.UpdRound, &rnd, &buf) + if err != nil { + return err + } + err = protocol.Decode(buf, &data.AccountData) + if err != nil { + return err + } + data.Addr = addr + result = append(result, data) + } + return err + }) + return +} + +func (qs *accountsDbQueries) Close() { + preparedQueries := []**sql.Stmt{ + &qs.listCreatablesStmt, + &qs.lookupAccountStmt, + &qs.lookupResourcesStmt, + &qs.lookupAllResourcesStmt, + &qs.lookupKvPairStmt, + &qs.lookupKeysByRangeStmt, + &qs.lookupCreatorStmt, + } + for _, preparedQuery := range preparedQueries { + if (*preparedQuery) != nil { + (*preparedQuery).Close() + *preparedQuery = nil + } + } +} + +func (qs *onlineAccountsDbQueries) Close() { + preparedQueries := []**sql.Stmt{ + &qs.lookupOnlineStmt, + &qs.lookupOnlineHistoryStmt, + } + for _, preparedQuery := range preparedQueries { + if (*preparedQuery) != nil { + (*preparedQuery).Close() + *preparedQuery = nil + } + } +} + +func (w *accountsSQLWriter) Close() { + // Formatted to match the type definition above + preparedStmts := []**sql.Stmt{ + &w.insertCreatableIdxStmt, &w.deleteCreatableIdxStmt, + &w.deleteByRowIDStmt, &w.insertStmt, &w.updateStmt, + &w.deleteResourceStmt, &w.insertResourceStmt, &w.updateResourceStmt, + &w.deleteKvPairStmt, &w.upsertKvPairStmt, + } + + for _, stmt := range preparedStmts { + if (*stmt) != nil { + (*stmt).Close() + *stmt = nil + } + } + +} + +func (w *onlineAccountsSQLWriter) Close() { + if w.insertStmt != nil { + w.insertStmt.Close() + w.insertStmt = nil + } +} + +func (w accountsSQLWriter) InsertAccount(addr basics.Address, normBalance uint64, data BaseAccountData) (rowid int64, err error) { + result, err := w.insertStmt.Exec(addr[:], normBalance, protocol.Encode(&data)) + if err != nil { + return + } + rowid, err = result.LastInsertId() + return +} + +func (w accountsSQLWriter) DeleteAccount(rowid int64) (rowsAffected int64, err error) { + result, err := w.deleteByRowIDStmt.Exec(rowid) + if err != nil { + return + } + rowsAffected, err = result.RowsAffected() + return +} + +func (w accountsSQLWriter) UpdateAccount(rowid int64, normBalance uint64, data BaseAccountData) (rowsAffected int64, err error) { + result, err := w.updateStmt.Exec(normBalance, protocol.Encode(&data), rowid) + if err != nil { + return + } + rowsAffected, err = result.RowsAffected() + return +} + +func (w accountsSQLWriter) InsertResource(addrid int64, aidx basics.CreatableIndex, data ResourcesData) (rowid int64, err error) { + result, err := w.insertResourceStmt.Exec(addrid, aidx, protocol.Encode(&data)) + if err != nil { + return + } + rowid, err = result.LastInsertId() + return +} + +func (w accountsSQLWriter) DeleteResource(addrid int64, aidx basics.CreatableIndex) (rowsAffected int64, err error) { + result, err := w.deleteResourceStmt.Exec(addrid, aidx) + if err != nil { + return + } + rowsAffected, err = result.RowsAffected() + return +} + +func (w accountsSQLWriter) UpdateResource(addrid int64, aidx basics.CreatableIndex, data ResourcesData) (rowsAffected int64, err error) { + result, err := w.updateResourceStmt.Exec(protocol.Encode(&data), addrid, aidx) + if err != nil { + return + } + rowsAffected, err = result.RowsAffected() + return +} + +func (w accountsSQLWriter) UpsertKvPair(key string, value []byte) error { + // NOTE! If we are passing in `string`, then for `BoxKey` case, + // we might contain 0-byte in boxKey, coming from uint64 appID. + // The consequence would be DB key write in be cut off after such 0-byte. + // Casting `string` to `[]byte` avoids such trouble, and test: + // - `TestBoxNamesByAppIDs` in `acctupdates_test` + // relies on such modification. + result, err := w.upsertKvPairStmt.Exec([]byte(key), value) + if err != nil { + return err + } + _, err = result.LastInsertId() + return err +} + +func (w accountsSQLWriter) DeleteKvPair(key string) error { + // Cast to []byte to avoid interpretation as character string, see note in upsertKvPair + result, err := w.deleteKvPairStmt.Exec([]byte(key)) + if err != nil { + return err + } + _, err = result.RowsAffected() + return err +} + +func (w accountsSQLWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (rowid int64, err error) { + result, err := w.insertCreatableIdxStmt.Exec(cidx, creator, ctype) + if err != nil { + return + } + rowid, err = result.LastInsertId() + return +} + +func (w accountsSQLWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { + result, err := w.deleteCreatableIdxStmt.Exec(cidx, ctype) + if err != nil { + return + } + rowsAffected, err = result.RowsAffected() + return +} + +func (w onlineAccountsSQLWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (rowid int64, err error) { + result, err := w.insertStmt.Exec(addr[:], normBalance, protocol.Encode(&data), updRound, voteLastValid) + if err != nil { + return + } + rowid, err = result.LastInsertId() + return +} diff --git a/ledger/store/sql_test.go b/ledger/store/sql_test.go new file mode 100644 index 0000000000..2eb96bfe65 --- /dev/null +++ b/ledger/store/sql_test.go @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package store + +import ( + "testing" + + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" +) + +func TestKeyPrefixIntervalPreprocessing(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + testCases := []struct { + input []byte + outputPrefix []byte + outputPrefixIncr []byte + }{ + {input: []byte{0xAB, 0xCD}, outputPrefix: []byte{0xAB, 0xCD}, outputPrefixIncr: []byte{0xAB, 0xCE}}, + {input: []byte{0xFF}, outputPrefix: []byte{0xFF}, outputPrefixIncr: nil}, + {input: []byte{0xFE, 0xFF}, outputPrefix: []byte{0xFE, 0xFF}, outputPrefixIncr: []byte{0xFF}}, + {input: []byte{0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFF}, outputPrefixIncr: nil}, + {input: []byte{0xAB, 0xCD}, outputPrefix: []byte{0xAB, 0xCD}, outputPrefixIncr: []byte{0xAB, 0xCE}}, + {input: []byte{0x1E, 0xFF, 0xFF}, outputPrefix: []byte{0x1E, 0xFF, 0xFF}, outputPrefixIncr: []byte{0x1F}}, + {input: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefixIncr: []byte{0xFF, 0xFF}}, + {input: []byte{0x00, 0xFF}, outputPrefix: []byte{0x00, 0xFF}, outputPrefixIncr: []byte{0x01}}, + {input: []byte(string("bx:123")), outputPrefix: []byte(string("bx:123")), outputPrefixIncr: []byte(string("bx:124"))}, + {input: []byte{}, outputPrefix: []byte{}, outputPrefixIncr: nil}, + {input: nil, outputPrefix: []byte{}, outputPrefixIncr: nil}, + {input: []byte{0x1E, 0xFF, 0xFF}, outputPrefix: []byte{0x1E, 0xFF, 0xFF}, outputPrefixIncr: []byte{0x1F}}, + {input: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefix: []byte{0xFF, 0xFE, 0xFF, 0xFF}, outputPrefixIncr: []byte{0xFF, 0xFF}}, + {input: []byte{0x00, 0xFF}, outputPrefix: []byte{0x00, 0xFF}, outputPrefixIncr: []byte{0x01}}, + } + for _, tc := range testCases { + actualOutputPrefix, actualOutputPrefixIncr := keyPrefixIntervalPreprocessing(tc.input) + require.Equal(t, tc.outputPrefix, actualOutputPrefix) + require.Equal(t, tc.outputPrefixIncr, actualOutputPrefixIncr) + } +} diff --git a/ledger/store/testing/helpers.go b/ledger/store/testing/helpers.go new file mode 100644 index 0000000000..0fb5ec8246 --- /dev/null +++ b/ledger/store/testing/helpers.go @@ -0,0 +1,43 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testing + +import ( + "fmt" + "strings" + "testing" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/util/db" + "github.com/stretchr/testify/require" +) + +// DbOpenTest opens a db file for testing purposes. +func DbOpenTest(t testing.TB, inMemory bool) (db.Pair, string) { + fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) + dbs, err := db.OpenPair(fn, inMemory) + require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory) + return dbs, fn +} + +// SetDbLogging sets a testing logger on a database. +func SetDbLogging(t testing.TB, dbs db.Pair) { + dblogger := logging.TestingLog(t) + dbs.Rdb.SetLogger(dblogger) + dbs.Wdb.SetLogger(dblogger) +} diff --git a/ledger/tracker.go b/ledger/tracker.go index 1945018efb..94d2dbd9c3 100644 --- a/ledger/tracker.go +++ b/ledger/tracker.go @@ -31,6 +31,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/internal" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" @@ -246,12 +247,12 @@ type deferredCommitContext struct { compactKvDeltas map[string]modifiedKvValue compactCreatableDeltas map[basics.CreatableIndex]ledgercore.ModifiedCreatable - updatedPersistedAccounts []persistedAccountData - updatedPersistedResources map[basics.Address][]persistedResourcesData - updatedPersistedKVs map[string]persistedKVData + updatedPersistedAccounts []store.PersistedAccountData + updatedPersistedResources map[basics.Address][]store.PersistedResourcesData + updatedPersistedKVs map[string]store.PersistedKVData compactOnlineAccountDeltas compactOnlineAccountDeltas - updatedPersistedOnlineAccounts []persistedOnlineAccountData + updatedPersistedOnlineAccounts []store.PersistedOnlineAccountData updatingBalancesDuration time.Duration