Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

exp/orderbook: Represent assets in orderbook graph as int32 instead of strings #4102

Merged
merged 4 commits into from
Nov 30, 2021
Merged

exp/orderbook: Represent assets in orderbook graph as int32 instead of strings #4102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
06eebae
Represent assets in orderbook graph as int32 instead of strings
tamirms Nov 24, 2021
41b735e
Add comments
tamirms Nov 29, 2021
0e78847
Merge branch 'master' into int32-nodes
tamirms Nov 29, 2021
ad9c53f
Add tests for path resource adapter
tamirms Nov 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions exp/orderbook/edges.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ import (
"github.com/stellar/go/xdr"
)

// edgeSet maintains a mapping of strings (asset keys) to a set of venues, which
// edgeSet maintains a mapping of assets to a set of venues, which
// is composed of a sorted lists of offers and, optionally, a liquidity pool.
// The offers are sorted by ascending price (in terms of the buying asset).
type edgeSet []edge

type edge struct {
key string
key int32
value Venues
}

func (e edgeSet) find(key string) int {
func (e edgeSet) find(key int32) int {
for i := 0; i < len(e); i++ {
if e[i].key == key {
return i
Expand All @@ -26,7 +26,7 @@ func (e edgeSet) find(key string) int {
}

// addOffer will insert the given offer into the edge set
func (e edgeSet) addOffer(key string, offer xdr.OfferEntry) edgeSet {
func (e edgeSet) addOffer(key int32, offer xdr.OfferEntry) edgeSet {
// The list of offers in a venue is sorted by cheapest to most expensive
// price to convert buyingAsset to sellingAsset
i := e.find(key)
Expand All @@ -51,7 +51,7 @@ func (e edgeSet) addOffer(key string, offer xdr.OfferEntry) edgeSet {
}

// addPool makes `pool` a viable venue at `key`.
func (e edgeSet) addPool(key string, pool xdr.LiquidityPoolEntry) edgeSet {
func (e edgeSet) addPool(key int32, pool liquidityPool) edgeSet {
i := e.find(key)
if i < 0 {
return append(e, edge{key: key, value: Venues{pool: pool}})
Expand All @@ -62,7 +62,7 @@ func (e edgeSet) addPool(key string, pool xdr.LiquidityPoolEntry) edgeSet {

// removeOffer will delete the given offer from the edge set, returning whether
// or not the given offer was actually found.
func (e edgeSet) removeOffer(key string, offerID xdr.Int64) (edgeSet, bool) {
func (e edgeSet) removeOffer(key int32, offerID xdr.Int64) (edgeSet, bool) {
i := e.find(key)
if i < 0 {
return e, false
Expand Down Expand Up @@ -94,7 +94,7 @@ func (e edgeSet) removeOffer(key string, offerID xdr.Int64) (edgeSet, bool) {
return e, true
}

func (e edgeSet) removePool(key string) edgeSet {
func (e edgeSet) removePool(key int32) edgeSet {
i := e.find(key)
if i < 0 {
return e
Expand Down
168 changes: 117 additions & 51 deletions exp/orderbook/graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ const (
// trading pair represents two assets that can be exchanged if an order is fulfilled
type tradingPair struct {
// buyingAsset corresponds to offer.Buying.String() from an xdr.OfferEntry
buyingAsset string
buyingAsset int32
// sellingAsset corresponds to offer.Selling.String() from an xdr.OfferEntry
sellingAsset string
sellingAsset int32
}

// OBGraph is an interface for orderbook graphs
Expand All @@ -49,12 +49,26 @@ type OBGraph interface {
// OrderBookGraph is an in-memory graph representation of all the offers in the
// Stellar ledger.
type OrderBookGraph struct {
// idToAssetString maps an int32 asset id to its string representation.
// Every asset on the OrderBookGraph has an int32 id which indexes into idToAssetString.
// The asset integer ids are largely contiguous. When an asset is completely removed
// from the OrderBookGraph the integer id for that asset will be assigned to the next
// asset which is added to the OrderBookGraph.
idToAssetString []string
// assetStringToID maps an asset string to its int32 id.
assetStringToID map[string]int32
// vacantIDs is a list of int32 asset ids which can be mapped to new assets.
// When a new asset is added to the OrderBookGraph we first check if there are
// any available vacantIDs, if so, we will assign the new asset to one of the vacantIDs.
// Otherwise, we will add a new entry to idToAssetString for the new asset.
vacantIDs []int32

// venuesForBuyingAsset maps an asset to all of its buying opportunities,
// which may be offers (sorted by price) or a liquidity pools.
venuesForBuyingAsset map[string]edgeSet
// venuesForBuyingAsset maps an asset to all of its *selling* opportunities,
venuesForBuyingAsset []edgeSet
// venuesForSellingAsset maps an asset to all of its *selling* opportunities,
// which may be offers (sorted by price) or a liquidity pools.
venuesForSellingAsset map[string]edgeSet
venuesForSellingAsset []edgeSet
// liquidityPools associates a particular asset pair (in "asset order", see
// xdr.Asset.LessThan) with a liquidity pool.
liquidityPools map[tradingPair]xdr.LiquidityPoolEntry
Expand Down Expand Up @@ -169,8 +183,11 @@ func (graph *OrderBookGraph) Clear() {
graph.lock.Lock()
defer graph.lock.Unlock()

graph.venuesForBuyingAsset = map[string]edgeSet{}
graph.venuesForSellingAsset = map[string]edgeSet{}
graph.assetStringToID = map[string]int32{}
graph.idToAssetString = []string{}
graph.vacantIDs = []int32{}
graph.venuesForSellingAsset = []edgeSet{}
graph.venuesForBuyingAsset = []edgeSet{}
graph.tradingPairForOffer = map[xdr.Int64]tradingPair{}
graph.liquidityPools = map[tradingPair]xdr.LiquidityPoolEntry{}
graph.batchedUpdates = graph.batch()
Expand All @@ -187,6 +204,47 @@ func (graph *OrderBookGraph) batch() *orderBookBatchedUpdates {
}
}

func (graph *OrderBookGraph) getOrCreateAssetID(asset xdr.Asset) int32 {
assetString := asset.String()
id, ok := graph.assetStringToID[assetString]
if ok {
return id
}
// before creating a new int32 asset id we will try to use
// a vacant id so that we can plug any empty cells in the
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

great design, just curious, would storing nil in idToAssetString equate to same result as maintaining separate vacancy state, i.e., iterate for idToAssetString=nil instead, perhaps for less code, but just wondering.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, that's true we could avoid having a vacantIDs list entirely if we scan through idToAssetString to find the first empty cell. in the worst case if there are no empty cells we have to scan through the entire array before realizing we have to append to the end. Having vacantIDs makes the operation of adding a new asset faster

// idToAssetString array.
if len(graph.vacantIDs) > 0 {
id = graph.vacantIDs[len(graph.vacantIDs)-1]
graph.vacantIDs = graph.vacantIDs[:len(graph.vacantIDs)-1]
graph.idToAssetString[id] = assetString
} else {
// idToAssetString never decreases in length unless we call graph.Clear()
id = int32(len(graph.idToAssetString))
tamirms marked this conversation as resolved.
Show resolved Hide resolved
// we assign id to asset
graph.idToAssetString = append(graph.idToAssetString, assetString)
graph.venuesForBuyingAsset = append(graph.venuesForBuyingAsset, nil)
graph.venuesForSellingAsset = append(graph.venuesForSellingAsset, nil)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we clear venuesForBuyingAsset and venuesForSellingAsset when assigning to a vacant id? It seems we don't do that in maybeDeleteAsset either.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in order to get included in the vacant id list it is a necessary condition that graph.venuesForBuyingAsset[asset] and graph.venuesForSellingAsset[asset] are empty:

func (graph *OrderBookGraph) maybeDeleteAsset(asset int32) {
	buyingEdgesEmpty := len(graph.venuesForBuyingAsset[asset]) == 0
	sellingEdgesEmpty := len(graph.venuesForSellingAsset[asset]) == 0

	if buyingEdgesEmpty && sellingEdgesEmpty {
		delete(graph.assetStringToID, graph.idToAssetString[asset])
		// When removing an asset we do not resize the idToAssetString array.
		// Instead, we allow the cell occupied by the id to be empty.
		// The next time we will add an asset to the graph we will allocate the
		// id to the new asset.
		graph.idToAssetString[asset] = ""
		graph.vacantIDs = append(graph.vacantIDs, asset)
	}
}

}

graph.assetStringToID[assetString] = id
return id
}

func (graph *OrderBookGraph) maybeDeleteAsset(asset int32) {
buyingEdgesEmpty := len(graph.venuesForBuyingAsset[asset]) == 0
sellingEdgesEmpty := len(graph.venuesForSellingAsset[asset]) == 0

if buyingEdgesEmpty && sellingEdgesEmpty {
delete(graph.assetStringToID, graph.idToAssetString[asset])
// When removing an asset we do not resize the idToAssetString array.
// Instead, we allow the cell occupied by the id to be empty.
// The next time we will add an asset to the graph we will allocate the
// id to the new asset.
graph.idToAssetString[asset] = ""
graph.vacantIDs = append(graph.vacantIDs, asset)
}
}

// addOffer inserts a given offer into the order book graph
func (graph *OrderBookGraph) addOffer(offer xdr.OfferEntry) error {
// If necessary, replace any existing offer with a new one.
Expand All @@ -196,7 +254,8 @@ func (graph *OrderBookGraph) addOffer(offer xdr.OfferEntry) error {
}
}

buying, selling := offer.Buying.String(), offer.Selling.String()
buying := graph.getOrCreateAssetID(offer.Buying)
selling := graph.getOrCreateAssetID(offer.Selling)

graph.tradingPairForOffer[offer.OfferId] = tradingPair{
buyingAsset: buying, sellingAsset: selling,
Expand All @@ -208,19 +267,32 @@ func (graph *OrderBookGraph) addOffer(offer xdr.OfferEntry) error {
return nil
}

func (graph *OrderBookGraph) poolFromEntry(poolXDR xdr.LiquidityPoolEntry) liquidityPool {
aXDR, bXDR := getPoolAssets(poolXDR)
assetA, assetB := graph.getOrCreateAssetID(aXDR), graph.getOrCreateAssetID(bXDR)
return liquidityPool{
LiquidityPoolEntry: poolXDR,
assetA: assetA,
assetB: assetB,
}
}

// addPool sets the given pool as the venue for the given trading pair.
func (graph *OrderBookGraph) addPool(pool xdr.LiquidityPoolEntry) {
func (graph *OrderBookGraph) addPool(poolEntry xdr.LiquidityPoolEntry) {
// Liquidity pools have no concept of a "buying" or "selling" asset,
// so we create venues in both directions.
x, y := getPoolAssets(pool)
graph.liquidityPools[tradingPair{x, y}] = pool
pool := graph.poolFromEntry(poolEntry)
graph.liquidityPools[tradingPair{
buyingAsset: pool.assetA,
sellingAsset: pool.assetB,
}] = pool.LiquidityPoolEntry

for _, table := range []map[string]edgeSet{
for _, table := range [][]edgeSet{
graph.venuesForBuyingAsset,
graph.venuesForSellingAsset,
} {
table[x] = table[x].addPool(y, pool)
table[y] = table[y].addPool(x, pool)
table[pool.assetA] = table[pool.assetA].addPool(pool.assetB, pool)
table[pool.assetB] = table[pool.assetB].addPool(pool.assetA, pool)
}
}

Expand All @@ -230,59 +302,55 @@ func (graph *OrderBookGraph) removeOffer(offerID xdr.Int64) error {
if !ok {
return errOfferNotPresent
}

delete(graph.tradingPairForOffer, offerID)

if set, ok := graph.venuesForSellingAsset[pair.sellingAsset]; !ok {
return errOfferNotPresent
} else if set, ok = set.removeOffer(pair.buyingAsset, offerID); !ok {
if set, ok := graph.venuesForSellingAsset[pair.sellingAsset].removeOffer(pair.buyingAsset, offerID); !ok {
return errOfferNotPresent
} else if len(set) == 0 {
delete(graph.venuesForSellingAsset, pair.sellingAsset)
} else {
graph.venuesForSellingAsset[pair.sellingAsset] = set
}

if set, ok := graph.venuesForBuyingAsset[pair.buyingAsset]; !ok {
if set, ok := graph.venuesForBuyingAsset[pair.buyingAsset].removeOffer(pair.sellingAsset, offerID); !ok {
return errOfferNotPresent
} else if set, ok = set.removeOffer(pair.sellingAsset, offerID); !ok {
return errOfferNotPresent
} else if len(set) == 0 {
delete(graph.venuesForBuyingAsset, pair.buyingAsset)
} else {
graph.venuesForBuyingAsset[pair.buyingAsset] = set
}

graph.maybeDeleteAsset(pair.buyingAsset)
graph.maybeDeleteAsset(pair.sellingAsset)
return nil
}

// removePool unsets the pool matching the given asset pair, if it exists.
func (graph *OrderBookGraph) removePool(pool xdr.LiquidityPoolEntry) {
x, y := getPoolAssets(pool)

for _, asset := range []string{x, y} {
otherAsset := x
if asset == x {
otherAsset = y
func (graph *OrderBookGraph) removePool(poolXDR xdr.LiquidityPoolEntry) {
aXDR, bXDR := getPoolAssets(poolXDR)
assetA, assetB := graph.getOrCreateAssetID(aXDR), graph.getOrCreateAssetID(bXDR)

for _, asset := range []int32{assetA, assetB} {
otherAsset := assetB
if asset == assetB {
otherAsset = assetA
}

for _, table := range []map[string]edgeSet{
for _, table := range [][]edgeSet{
graph.venuesForBuyingAsset,
graph.venuesForSellingAsset,
} {
table[asset] = table[asset].removePool(otherAsset)
}
}

delete(graph.liquidityPools, tradingPair{x, y})
delete(graph.liquidityPools, tradingPair{assetA, assetB})
graph.maybeDeleteAsset(assetA)
graph.maybeDeleteAsset(assetB)
}

// IsEmpty returns true if the orderbook graph is not populated
func (graph *OrderBookGraph) IsEmpty() bool {
graph.lock.RLock()
defer graph.lock.RUnlock()

return len(graph.venuesForSellingAsset) == 0
return len(graph.liquidityPools) == 0 && len(graph.tradingPairForOffer) == 0
}

// FindPaths returns a list of payment paths originating from a source account
Expand All @@ -300,15 +368,15 @@ func (graph *OrderBookGraph) FindPaths(
includePools bool,
) ([]Path, uint32, error) {
destinationAssetString := destinationAsset.String()
sourceAssetsMap := make(map[string]xdr.Int64, len(sourceAssets))
sourceAssetsMap := make(map[int32]xdr.Int64, len(sourceAssets))
for i, sourceAsset := range sourceAssets {
sourceAssetString := sourceAsset.String()
sourceAssetsMap[sourceAssetString] = sourceAssetBalances[i]
sourceAssetsMap[graph.assetStringToID[sourceAssetString]] = sourceAssetBalances[i]
}

searchState := &sellingGraphSearchState{
graph: graph,
destinationAsset: destinationAsset,
destinationAssetString: destinationAssetString,
destinationAssetAmount: destinationAmount,
ignoreOffersFrom: sourceAccountID,
targetAssets: sourceAssetsMap,
Expand All @@ -321,8 +389,7 @@ func (graph *OrderBookGraph) FindPaths(
ctx,
searchState,
maxPathLength,
destinationAssetString,
destinationAsset,
graph.assetStringToID[destinationAssetString],
destinationAmount,
)
lastLedger := graph.lastLedger
Expand Down Expand Up @@ -356,15 +423,15 @@ func (graph *OrderBookGraph) FindFixedPaths(
maxAssetsPerPath int,
includePools bool,
) ([]Path, uint32, error) {
target := map[string]bool{}
target := map[int32]bool{}
for _, destinationAsset := range destinationAssets {
destinationAssetString := destinationAsset.String()
target[destinationAssetString] = true
target[graph.assetStringToID[destinationAssetString]] = true
}

searchState := &buyingGraphSearchState{
graph: graph,
sourceAsset: sourceAsset,
sourceAssetString: sourceAsset.String(),
sourceAssetAmount: amountToSpend,
targetAssets: target,
paths: []Path{},
Expand All @@ -375,8 +442,7 @@ func (graph *OrderBookGraph) FindFixedPaths(
ctx,
searchState,
maxPathLength,
sourceAsset.String(),
sourceAsset,
graph.assetStringToID[sourceAsset.String()],
amountToSpend,
)
lastLedger := graph.lastLedger
Expand All @@ -402,35 +468,35 @@ func (graph *OrderBookGraph) FindFixedPaths(
// if there are multiple paths which spend the same `SourceAmount` then shorter payment paths
// will be prioritized
func compareSourceAsset(allPaths []Path, i, j int) bool {
if allPaths[i].SourceAsset.Equals(allPaths[j].SourceAsset) {
if allPaths[i].SourceAsset == allPaths[j].SourceAsset {
if allPaths[i].SourceAmount == allPaths[j].SourceAmount {
return len(allPaths[i].InteriorNodes) < len(allPaths[j].InteriorNodes)
}
return allPaths[i].SourceAmount < allPaths[j].SourceAmount
}
return allPaths[i].SourceAssetString() < allPaths[j].SourceAssetString()
return allPaths[i].SourceAsset < allPaths[j].SourceAsset
}

// compareDestinationAsset will group payment paths by `DestinationAsset`. Paths
// which deliver a higher `DestinationAmount` will appear earlier in the
// sorting. If there are multiple paths which deliver the same
// `DestinationAmount`, then shorter payment paths will be prioritized.
func compareDestinationAsset(allPaths []Path, i, j int) bool {
if allPaths[i].DestinationAsset.Equals(allPaths[j].DestinationAsset) {
if allPaths[i].DestinationAsset == allPaths[j].DestinationAsset {
if allPaths[i].DestinationAmount == allPaths[j].DestinationAmount {
return len(allPaths[i].InteriorNodes) < len(allPaths[j].InteriorNodes)
}
return allPaths[i].DestinationAmount > allPaths[j].DestinationAmount
}
return allPaths[i].DestinationAssetString() < allPaths[j].DestinationAssetString()
return allPaths[i].DestinationAsset < allPaths[j].DestinationAsset
}

func sourceAssetEquals(p, otherPath Path) bool {
return p.SourceAsset.Equals(otherPath.SourceAsset)
return p.SourceAsset == otherPath.SourceAsset
}

func destinationAssetEquals(p, otherPath Path) bool {
return p.DestinationAsset.Equals(otherPath.DestinationAsset)
return p.DestinationAsset == otherPath.DestinationAsset
}

// sortAndFilterPaths sorts the given list of paths using `comparePaths`
Expand Down
Loading