amend bitflow scraper suite for deployment.

diadata-org · Oct 14, 2024 · 4398c41 · 4398c41
1 parent 71095cb
commit 4398c41
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 14 deletions.
diff --git a/cmd/assetCollectionService/go.mod b/cmd/assetCollectionService/go.mod
@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService
 go 1.20
 
 require (
-	github.com/diadata-org/diadata v1.4.539
+	github.com/diadata-org/diadata v1.4.543
 	github.com/sirupsen/logrus v1.9.3
 )
 

diff --git a/cmd/exchange-scrapers/collector/collector.go b/cmd/exchange-scrapers/collector/collector.go
@@ -42,6 +42,7 @@ var (
 		dia.ZenlinkswapExchangeBifrostPolkadot,
 		dia.PearlfiExchangeTestnet,
 		dia.PanCakeSwapExchangeV3,
+		dia.BitflowExchange,
 	}
 
 	exchange = flag.String("exchange", "", "which exchange")

diff --git a/cmd/exchange-scrapers/collector/go.mod b/cmd/exchange-scrapers/collector/go.mod
@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/exchange-scrapers/collector
 go 1.19
 
 require (
-	github.com/diadata-org/diadata v1.4.541
+	github.com/diadata-org/diadata v1.4.543
 	github.com/segmentio/kafka-go v0.4.35
 	github.com/sirupsen/logrus v1.9.3
 )

diff --git a/cmd/liquidityScraper/go.mod b/cmd/liquidityScraper/go.mod
@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/liquidityScraper
 go 1.20
 
 require (
-	github.com/diadata-org/diadata v1.4.539
+	github.com/diadata-org/diadata v1.4.543
 	github.com/sirupsen/logrus v1.9.3
 )
 

diff --git a/pkg/dia/scraper/exchange-scrapers/BitflowScraper.go b/pkg/dia/scraper/exchange-scrapers/BitflowScraper.go
@@ -14,6 +14,7 @@ import (
 	models "github.com/diadata-org/diadata/pkg/model"
 	"github.com/diadata-org/diadata/pkg/utils"
 	"github.com/sirupsen/logrus"
+	"github.com/zekroTJA/timedmap"
 )
 
 const DefaultRefreshDelay = 10000 // ms
@@ -36,7 +37,7 @@ type BitflowScraper struct {
 	currentHeight int
 }
 
-// NewAyinScraper returns a new AyinScraper initialized with default values.
+// NewBitflowScraper returns a new BitflowScraper initialized with default values.
 // The instance is asynchronously scraping as soon as it is created.
 // ENV values:
 //
@@ -150,6 +151,8 @@ func (s *BitflowScraper) Update() error {
 	}
 
 	for _, pool := range pools {
+		tmFalseDuplicateTrades := timedmap.New(duplicateTradesScanFrequency)
+		tmDuplicateTrades := timedmap.New(duplicateTradesScanFrequency)
 		if len(pool.Assetvolumes) != 2 {
 			s.logger.WithField("poolAddress", pool.Address).Error("pool is missing required asset volumes")
 			continue
@@ -168,12 +171,16 @@ func (s *BitflowScraper) Update() error {
 			}
 
 			diaTrade := s.handleTrade(&pool, &tx)
-			s.logger.
-				WithField("parentAddress", pool.Address).
-				WithField("height", s.currentHeight).
-				WithField("diaTrade", diaTrade).
-				Info("trade")
-			s.chanTrades <- diaTrade
+			log.Infof("got trade at height %v: %v -- %s -- %v --%v -- %s", s.currentHeight, diaTrade.Time, diaTrade.Pair, diaTrade.Price, diaTrade.Volume, diaTrade.ForeignTradeID)
+			discardTrade := diaTrade.IdentifyDuplicateFull(tmFalseDuplicateTrades, duplicateTradesMemory)
+			if discardTrade {
+				log.Warn("Identical trade already scraped: ", diaTrade)
+				continue
+			} else {
+				diaTrade.IdentifyDuplicateTagset(tmDuplicateTrades, duplicateTradesMemory)
+				s.chanTrades <- diaTrade
+			}
+
 		}
 	}
 
@@ -242,15 +249,15 @@ func (s *BitflowScraper) handleTrade(pool *dia.Pool, tx *stackshelper.Transactio
 
 	return &dia.Trade{
 		Time:           time.Unix(int64(tx.BlockTime), 0),
-		Symbol:         symbolPair,
+		Symbol:         pool.Assetvolumes[0].Asset.Symbol,
 		Pair:           symbolPair,
 		ForeignTradeID: tx.TxID,
 		Source:         s.exchangeName,
 		Price:          price,
 		Volume:         volume,
 		VerifiedPair:   true,
-		BaseToken:      pool.Assetvolumes[0].Asset,
-		QuoteToken:     pool.Assetvolumes[1].Asset,
+		BaseToken:      pool.Assetvolumes[1].Asset,
+		QuoteToken:     pool.Assetvolumes[0].Asset,
 		PoolAddress:    pool.Address,
 	}
 }

diff --git a/pkg/dia/scraper/liquidity-scrapers/BitflowScraper.go b/pkg/dia/scraper/liquidity-scrapers/BitflowScraper.go
@@ -118,7 +118,7 @@ func (s *BitflowLiquidityScraper) fetchPools() {
 			// Workaround to fetch the native STX token data from DB
 			key := address
 			if address == "" {
-				key = "null"
+				key = "0x0000000000000000000000000000000000000000"
 			}
 
 			assset, err := s.relDB.GetAsset(key, s.blockchain)

diff --git a/pkg/dia/service/assetservice/source/bitflow.go b/pkg/dia/service/assetservice/source/bitflow.go
@@ -70,6 +70,9 @@ func (s *BitflowAssetSource) fetchAssets() {
 	}
 
 	for _, token := range tokens {
+		if token.Symbol == "STX" {
+			token.TokenContract = "0x0000000000000000000000000000000000000000"
+		}
 		s.assetChannel <- dia.Asset{
 			Address:    token.TokenContract,
 			Name:       token.Name,