From d977162eb145d4276e3df85c81a81d5ca708b5e4 Mon Sep 17 00:00:00 2001 From: Jonathan Chappelow Date: Wed, 28 Apr 2021 11:30:30 -0500 Subject: [PATCH] excludeCancels option for db.MatchesForOrder and reloadMatchProofs Cancel matches can be the lion's share of matches in some DBs, and in most cases it is not necessary to process these. - In db upgrades, there is no reason to reencode the MatchProof. Although it's not clear there is a reason to do this for trade matches either since any match proof version decodes automatically to the current MatchProof. - In (*Core).dbTrackers, cancel order matches are not used. - (*Core).coreOrderFromMetaOrder might be able to exclude cancel order matches, but the consumers are exported Core methods, so no change. This also include a prealloc optional arg to ExtractPushes and DecodeBlob, which is mainly helpful for the MatchProof buckets that have 22 pushes. --- client/core/core.go | 6 +++-- client/core/core_test.go | 2 +- client/db/bolt/db.go | 19 ++++++++------ client/db/bolt/upgrades.go | 48 ++++++++++++++++++++++-------------- client/db/interface.go | 2 +- client/db/test/types_test.go | 5 +++- client/db/types.go | 32 +++++++++++++++--------- dex/encode/encode.go | 12 ++++++--- dex/order/serialize.go | 8 +++--- 9 files changed, 84 insertions(+), 50 deletions(-) diff --git a/client/core/core.go b/client/core/core.go index e6b00b2512..222fd70166 100644 --- a/client/core/core.go +++ b/client/core/core.go @@ -2548,7 +2548,8 @@ func (c *Core) Orders(filter *OrderFilter) ([]*Order, error) { func (c *Core) coreOrderFromMetaOrder(mOrd *db.MetaOrder) (*Order, error) { corder := coreOrderFromTrade(mOrd.Order, mOrd.MetaData) oid := mOrd.Order.ID() - matches, err := c.db.MatchesForOrder(oid) + excludeCancels := false // maybe don't include cancel order matches? + matches, err := c.db.MatchesForOrder(oid, excludeCancels) if err != nil { return nil, fmt.Errorf("MatchesForOrder error loading matches for %s: %w", oid, err) } @@ -3901,6 +3902,7 @@ func (c *Core) dbTrackers(dc *dexConnection) (map[order.OrderID]*trackedTrade, e } trackers := make(map[order.OrderID]*trackedTrade, len(dbOrders)) + excludeCancelMatches := true for _, dbOrder := range dbOrders { ord := dbOrder.Order oid := ord.ID() @@ -3923,7 +3925,7 @@ func (c *Core) dbTrackers(dc *dexConnection) (map[order.OrderID]*trackedTrade, e trackers[dbOrder.Order.ID()] = tracker // Get matches. - dbMatches, err := c.db.MatchesForOrder(oid) + dbMatches, err := c.db.MatchesForOrder(oid, excludeCancelMatches) if err != nil { return nil, fmt.Errorf("error loading matches for order %s: %w", oid, err) } diff --git a/client/core/core_test.go b/client/core/core_test.go index f119281a28..625b605dd5 100644 --- a/client/core/core_test.go +++ b/client/core/core_test.go @@ -386,7 +386,7 @@ func (tdb *TDB) ActiveMatches() ([]*db.MetaMatch, error) { return nil, nil } -func (tdb *TDB) MatchesForOrder(oid order.OrderID) ([]*db.MetaMatch, error) { +func (tdb *TDB) MatchesForOrder(oid order.OrderID, excludeCancels bool) ([]*db.MetaMatch, error) { return tdb.matchesForOID, tdb.matchesForOIDErr } diff --git a/client/db/bolt/db.go b/client/db/bolt/db.go index df66287c97..4ccf97259d 100644 --- a/client/db/bolt/db.go +++ b/client/db/bolt/db.go @@ -818,7 +818,7 @@ func (db *BoltDB) ActiveMatches() ([]*dexdb.MetaMatch, error) { return db.filteredMatches(func(mBkt *bbolt.Bucket) bool { status := mBkt.Get(statusKey) return len(status) != 1 || status[0] != uint8(order.MatchComplete) - }) + }, true) // cancel matches are immediately complete, never active, but the status filter will apply first anyway } // DEXOrdersWithActiveMatches retrieves order IDs for any order that has active @@ -853,7 +853,7 @@ func (db *BoltDB) DEXOrdersWithActiveMatches(dex string) ([]order.OrderID, error db.log.Errorf("empty match proof") return nil } - proof, errM := dexdb.DecodeMatchProof(proofB) + proof, _, errM := dexdb.DecodeMatchProof(proofB) if errM != nil { db.log.Errorf("error decoding proof: %v", errM) return nil @@ -912,18 +912,20 @@ func (db *BoltDB) DEXOrdersWithActiveMatches(dex string) ([]order.OrderID, error } // MatchesForOrder retrieves the matches for the specified order ID. -func (db *BoltDB) MatchesForOrder(oid order.OrderID) ([]*dexdb.MetaMatch, error) { +func (db *BoltDB) MatchesForOrder(oid order.OrderID, excludeCancels bool) ([]*dexdb.MetaMatch, error) { oidB := oid[:] return db.filteredMatches(func(mBkt *bbolt.Bucket) bool { oid := mBkt.Get(orderIDKey) return bytes.Equal(oid, oidB) - }) + }, excludeCancels) } // filteredMatches gets all matches that pass the provided filter function. Each // match's bucket is provided to the filter, and a boolean true return value -// indicates the match should be decoded and returned. -func (db *BoltDB) filteredMatches(filter func(*bbolt.Bucket) bool) ([]*dexdb.MetaMatch, error) { +// indicates the match should be decoded and returned. Matches with cancel +// orders may be excluded, a separate option so the filter function does not +// need to load and decode the matchKey value. +func (db *BoltDB) filteredMatches(filter func(*bbolt.Bucket) bool, excludeCancels bool) ([]*dexdb.MetaMatch, error) { var matches []*dexdb.MetaMatch return matches, db.matchesView(func(master *bbolt.Bucket) error { return master.ForEach(func(k, _ []byte) error { @@ -941,11 +943,14 @@ func (db *BoltDB) filteredMatches(filter func(*bbolt.Bucket) bool) ([]*dexdb.Met if err != nil { return fmt.Errorf("error decoding match %x: %w", k, err) } + if excludeCancels && match.Address == "" { + return nil + } proofB := getCopy(mBkt, proofKey) if len(proofB) == 0 { return fmt.Errorf("empty proof") } - proof, err = dexdb.DecodeMatchProof(proofB) + proof, _, err = dexdb.DecodeMatchProof(proofB) if err != nil { return fmt.Errorf("error decoding proof: %w", err) } diff --git a/client/db/bolt/upgrades.go b/client/db/bolt/upgrades.go index 5cc02a5f60..43af01be33 100644 --- a/client/db/bolt/upgrades.go +++ b/client/db/bolt/upgrades.go @@ -9,6 +9,7 @@ import ( dexdb "decred.org/dcrdex/client/db" "decred.org/dcrdex/dex/encode" + "decred.org/dcrdex/dex/order" "go.etcd.io/bbolt" ) @@ -84,28 +85,18 @@ func (db *BoltDB) upgradeDB() error { return fmt.Errorf("failed to backup DB prior to upgrade: %w", err) } - // Each upgrade its own tx. + // Each upgrade its own tx, otherwise bolt eats too much RAM. for i, upgrade := range upgrades[version:] { + newVersion := version + uint32(i) + 1 + db.log.Debugf("Upgrading to version %d...", newVersion) err = db.Update(func(tx *bbolt.Tx) error { - return doUpgrade(tx, upgrade, version+uint32(i)+1) + return doUpgrade(tx, upgrade, newVersion) }) if err != nil { return err } } return nil - - // All upgrades in a single tx. - // return db.Update(func(tx *bbolt.Tx) error { - // // Execute all necessary upgrades in order. - // for i, upgrade := range upgrades[version:] { - // err := doUpgrade(tx, upgrade, version+uint32(i)+1) - // if err != nil { - // return err - // } - // } - // return nil - // }) } // Get the currently stored DB version. @@ -141,8 +132,8 @@ func v1Upgrade(dbtx *bbolt.Tx) error { if bkt == nil { return fmt.Errorf("appBucket not found") } - - return reloadMatchProofs(dbtx) + skipCancels := true // cancel matches don't get revoked, only cancel orders + return reloadMatchProofs(dbtx, skipCancels) } // v2Upgrade adds a MaxFeeRate field to the OrderMetaData. The upgrade sets the @@ -187,7 +178,8 @@ func v3Upgrade(dbtx *bbolt.Tx) error { // Upgrade the match proof. We just have to retrieve and re-store the // buckets. The decoder will recognize the the old version and add the new // field. - return reloadMatchProofs(dbtx) + skipCancels := true // cancel matches have no tx data + return reloadMatchProofs(dbtx, skipCancels) } func ensureVersion(tx *bbolt.Tx, ver uint32) error { @@ -202,7 +194,10 @@ func ensureVersion(tx *bbolt.Tx, ver uint32) error { return nil } -func reloadMatchProofs(tx *bbolt.Tx) error { +// Note that reloadMatchProofs will rewrite the MatchProof with the current +// match proof encoding version. Thus, multiple upgrades in a row calling +// reloadMatchProofs may be no-ops. Matches with cancel orders may be skipped. +func reloadMatchProofs(tx *bbolt.Tx, skipCancels bool) error { matches := tx.Bucket(matchesBucket) return matches.ForEach(func(k, _ []byte) error { mBkt := matches.Bucket(k) @@ -213,10 +208,25 @@ func reloadMatchProofs(tx *bbolt.Tx) error { if len(proofB) == 0 { return fmt.Errorf("empty match proof") } - proof, err := dexdb.DecodeMatchProof(proofB) + proof, ver, err := dexdb.DecodeMatchProof(proofB) if err != nil { return fmt.Errorf("error decoding proof: %w", err) } + // No need to rewrite this if it was loaded from the current version. + if ver == dexdb.MatchProofVer { + return nil + } + // No Script, and MatchComplete status means this is a cancel match. + if skipCancels && len(proof.Script) == 0 { + statusB := mBkt.Get(statusKey) + if len(statusB) != 1 { + return fmt.Errorf("no match status") + } + if order.MatchStatus(statusB[0]) == order.MatchComplete { + return nil + } + } + err = mBkt.Put(proofKey, proof.Encode()) if err != nil { return fmt.Errorf("error re-storing match proof: %w", err) diff --git a/client/db/interface.go b/client/db/interface.go index 0552f5bea7..b9b0a2d474 100644 --- a/client/db/interface.go +++ b/client/db/interface.go @@ -87,7 +87,7 @@ type DB interface { // state. DEXOrdersWithActiveMatches(dex string) ([]order.OrderID, error) // MatchesForOrder gets the matches for the order ID. - MatchesForOrder(oid order.OrderID) ([]*MetaMatch, error) + MatchesForOrder(oid order.OrderID, excludeCancels bool) ([]*MetaMatch, error) // Update wallets adds a wallet to the database, or updates the wallet // credentials if the wallet already exists. A wallet is specified by the // pair (asset ID, account name). diff --git a/client/db/test/types_test.go b/client/db/test/types_test.go index d91f8d7067..759d1fac58 100644 --- a/client/db/test/types_test.go +++ b/client/db/test/types_test.go @@ -40,11 +40,14 @@ func TestMatchProof(t *testing.T) { nTimes(spins, func(i int) { proof := proofs[i] proofB := proof.Encode() - reProof, err := db.DecodeMatchProof(proofB) + reProof, ver, err := db.DecodeMatchProof(proofB) if err != nil { t.Fatalf("match decode error: %v", err) } MustCompareMatchProof(t, proof, reProof) + if ver != db.MatchProofVer { + t.Errorf("wanted match proof ver %d, got %d", db.MatchProofVer, ver) + } }) t.Logf("encoded, decoded, and compared %d MatchProof in %d ms", spins, time.Since(tStart)/time.Millisecond) } diff --git a/client/db/types.go b/client/db/types.go index fd5f52f105..347a335321 100644 --- a/client/db/types.go +++ b/client/db/types.go @@ -254,6 +254,12 @@ type MatchProof struct { SelfRevoked bool } +// MatchProofVer is the current serialization version of a MatchProof. +const ( + MatchProofVer = 2 + matchProofPushes = 22 +) + // Encode encodes the MatchProof to a versioned blob. func (p *MatchProof) Encode() []byte { auth := p.Auth @@ -266,7 +272,7 @@ func (p *MatchProof) Encode() []byte { selfRevoked = encode.ByteTrue } - return dbBytes{2}. + return dbBytes{MatchProofVer}. AddData(p.Script). AddData(p.CounterContract). AddData(p.SecretHash). @@ -292,20 +298,23 @@ func (p *MatchProof) Encode() []byte { } // DecodeMatchProof decodes the versioned blob to a *MatchProof. -func DecodeMatchProof(b []byte) (*MatchProof, error) { - ver, pushes, err := encode.DecodeBlob(b) +func DecodeMatchProof(b []byte) (*MatchProof, uint8, error) { + ver, pushes, err := encode.DecodeBlob(b, matchProofPushes) if err != nil { - return nil, err + return nil, 0, err } switch ver { - case 2: - return decodeMatchProof_v2(pushes) + case 2: // MatchProofVer + proof, err := decodeMatchProof_v2(pushes) + return proof, ver, err case 1: - return decodeMatchProof_v1(pushes) + proof, err := decodeMatchProof_v1(pushes) + return proof, ver, err case 0: - return decodeMatchProof_v0(pushes) + proof, err := decodeMatchProof_v0(pushes) + return proof, ver, err } - return nil, fmt.Errorf("unknown MatchProof version %d", ver) + return nil, ver, fmt.Errorf("unknown MatchProof version %d", ver) } func decodeMatchProof_v0(pushes [][]byte) (*MatchProof, error) { @@ -319,8 +328,9 @@ func decodeMatchProof_v1(pushes [][]byte) (*MatchProof, error) { } func decodeMatchProof_v2(pushes [][]byte) (*MatchProof, error) { - if len(pushes) != 22 { - return nil, fmt.Errorf("DecodeMatchProof: expected 21 pushes, got %d", len(pushes)) + if len(pushes) != matchProofPushes { + return nil, fmt.Errorf("DecodeMatchProof: expected %d pushes, got %d", + matchProofPushes, len(pushes)) } return &MatchProof{ Script: pushes[0], diff --git a/dex/encode/encode.go b/dex/encode/encode.go index 3ce03ef470..7726015a7d 100644 --- a/dex/encode/encode.go +++ b/dex/encode/encode.go @@ -108,8 +108,12 @@ func DecodeUTime(b []byte) time.Time { // ExtractPushes parses the linearly-encoded 2D byte slice into a slice of // slices. Empty pushes are nil slices. -func ExtractPushes(b []byte) ([][]byte, error) { - pushes := make([][]byte, 0) +func ExtractPushes(b []byte, preAlloc ...int) ([][]byte, error) { + allocPushes := 2 + if len(preAlloc) > 0 { + allocPushes = preAlloc[0] + } + pushes := make([][]byte, 0, allocPushes) for { if len(b) == 0 { break @@ -139,13 +143,13 @@ func ExtractPushes(b []byte) ([][]byte, error) { // DecodeBlob decodes a versioned blob into its version and the pushes extracted // from its data. Empty pushes will be nil. -func DecodeBlob(b []byte) (byte, [][]byte, error) { +func DecodeBlob(b []byte, preAlloc ...int) (byte, [][]byte, error) { if len(b) == 0 { return 0, nil, fmt.Errorf("zero length blob not allowed") } ver := b[0] b = b[1:] - pushes, err := ExtractPushes(b) + pushes, err := ExtractPushes(b, preAlloc...) return ver, pushes, err } diff --git a/dex/order/serialize.go b/dex/order/serialize.go index fda46455ab..efae4f3409 100644 --- a/dex/order/serialize.go +++ b/dex/order/serialize.go @@ -31,7 +31,7 @@ func EncodePrefix(p *Prefix) []byte { // DecodePrefix decodes the versioned blob to a *Prefix. func DecodePrefix(b []byte) (prefix *Prefix, err error) { - ver, pushes, err := encode.DecodeBlob(b) + ver, pushes, err := encode.DecodeBlob(b, 7) if err != nil { return nil, err } @@ -98,7 +98,7 @@ func EncodeTrade(ord *Trade) []byte { // DecodeTrade decodes the versioned-blob market order, but does not populate // the embedded Prefix. func DecodeTrade(b []byte) (trade *Trade, err error) { - ver, pushes, err := encode.DecodeBlob(b) + ver, pushes, err := encode.DecodeBlob(b, 5) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func EncodeMatch(match *UserMatch) []byte { // DecodeMatch decodes the versioned blob into a UserMatch. func DecodeMatch(b []byte) (match *UserMatch, err error) { - ver, pushes, err := encode.DecodeBlob(b) + ver, pushes, err := encode.DecodeBlob(b, 8) if err != nil { return nil, err } @@ -252,7 +252,7 @@ func EncodeOrder(ord Order) []byte { // DecodeOrder decodes the byte-encoded order. DecodeOrder accepts any type of // order. func DecodeOrder(b []byte) (ord Order, err error) { - ver, pushes, err := encode.DecodeBlob(b) + ver, pushes, err := encode.DecodeBlob(b, 4) // pushes actually depends on order type if err != nil { return nil, err }