From ef978c21346e50f07ce6ed52cc6fedd7a3517d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Men=C3=A9ndez?= Date: Mon, 24 Jun 2024 11:11:23 +0200 Subject: [PATCH] final minor changes --- api/tokens.go | 18 +++++++++++++++--- db/queries/tokens.sql | 2 +- scanner/scanner.go | 11 +++++++++++ scanner/updater.go | 15 ++++++++++----- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/api/tokens.go b/api/tokens.go index f0f6ee81..45f5e022 100644 --- a/api/tokens.go +++ b/api/tokens.go @@ -45,11 +45,11 @@ func (capi *census3API) initTokenHandlers() error { api.MethodAccessTypePublic, capi.tokenStartBlock); err != nil { return err } - if err := capi.endpoint.RegisterMethod("/tokens/rescan/{tokenID}", "POST", + if err := capi.endpoint.RegisterMethod("/tokens/update/{tokenID}", "POST", api.MethodAccessTypeAdmin, capi.rescanToken); err != nil { return err } - if err := capi.endpoint.RegisterMethod("/tokens/rescan/queue/{queueID}", "GET", + if err := capi.endpoint.RegisterMethod("/tokens/update/queue/{queueID}", "GET", api.MethodAccessTypeAdmin, capi.checkRescanToken); err != nil { return err } @@ -617,6 +617,12 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext) return ctx.Send(res, api.HTTPstatusOK) } +// rescanToken function handler enqueues the rescan process for the token with +// the given ID. The token is scanned from the creation block to the last block +// stored in the database. It returns a 400 error if the provided ID is wrong or +// empty, a 404 error if the token is not found, a 500 error if something fails +// or a 200 response if the process is enqueued. It returns a queue ID to track +// the status of the process. func (capi *census3API) rescanToken(msg *api.APIdata, ctx *httprouter.HTTPContext) error { // get contract address from the tokenID query param and decode check if // it is provided, if not return an error @@ -664,7 +670,7 @@ func (capi *census3API) rescanToken(msg *api.APIdata, ctx *httprouter.HTTPContex CreationBlock: uint64(tokenData.CreationBlock), EndBlock: uint64(tokenData.LastBlock), }); err != nil { - return ErrMalformedToken.WithErr(err) + return ErrEncodeQueueItem.WithErr(err) } // encoding the result and response it res, err := json.Marshal(QueueResponse{id}) @@ -674,6 +680,12 @@ func (capi *census3API) rescanToken(msg *api.APIdata, ctx *httprouter.HTTPContex return ctx.Send(res, api.HTTPstatusOK) } +// checkRescanToken function handler returns the status of the rescan process +// with the given queue ID. It returns a 400 error if the provided ID is wrong +// or empty, a 404 error if the token is not found in the queue or a 500 error +// if something fails. The response contains the address of the token, the chain +// ID, the status of the process, the number of logs scanned, the number of new +// logs found, and the number of duplicated logs. func (capi *census3API) checkRescanToken(msg *api.APIdata, ctx *httprouter.HTTPContext) error { queueID := ctx.URLParam("queueID") if queueID == "" { diff --git a/db/queries/tokens.sql b/db/queries/tokens.sql index 3e419f82..f8ea29b4 100644 --- a/db/queries/tokens.sql +++ b/db/queries/tokens.sql @@ -112,4 +112,4 @@ FROM tokens WHERE id = ? AND chain_id = ? AND external_id = ? HAVING num_of_tokens = 1; -- name: DeleteToken :execresult -DELETE FROM tokens WHERE id = ? AND chain_id = ? AND external_id = ?; +DELETE FROM tokens WHERE id = ? AND chain_id = ? AND external_id = ?; \ No newline at end of file diff --git a/scanner/scanner.go b/scanner/scanner.go index a9a39310..aa8f2158 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -362,6 +362,12 @@ func (s *Scanner) getLatestBlockNumbersUpdates() { } } +// updateInternalTokenStatus updates the internal token status of the scanner +// with the given information. It is used to update the last block number and +// the synced status of the token in the scanner. It is used to avoid +// overloading the database with requests to get tokens information in every +// iteration of the scanner. It is used in the SaveHolders function to update +// the token status after saving the holders in the database. func (s *Scanner) updateInternalTokenStatus(token ScannerToken, lastBlock uint64, synced bool, totalSupply *big.Int, ) { @@ -380,6 +386,11 @@ func (s *Scanner) updateInternalTokenStatus(token ScannerToken, lastBlock uint64 s.tokensMtx.Unlock() } +// prepareToken prepares the token to be scanned. It calculates the creation +// block of the token if it is not ready yet. It updates the token in the +// scanner but also the token information in the database. It returns an error +// if something fails in the process. It sets the last block of the token to +// the creation block of the token to start scanning from the creation block. func (s *Scanner) prepareToken(token *ScannerToken) error { ctx, cancel := context.WithTimeout(s.ctx, UPDATE_TIMEOUT) defer cancel() diff --git a/scanner/updater.go b/scanner/updater.go index 8fec3cb3..d0c7885f 100644 --- a/scanner/updater.go +++ b/scanner/updater.go @@ -44,10 +44,7 @@ type UpdateRequest struct { // iterate over the requests, repeating the process of getting the token holders // balances and saving them in the database until the last block is greater or // equal to the end block. The end block is the block number where the token -// holders balances are up to date. The holders providers must include an -// instance of a TokenFilter to store the processed transactions to avoid -// re-processing them, but also rescanning a synced token to find missing -// transactions. +// holders balances are up to date. type Updater struct { ctx context.Context cancel context.CancelFunc @@ -113,10 +110,13 @@ func (u *Updater) Start(ctx context.Context, concurrentTokens int) { return } // update the request in the queue - log.Infow("updating request in the queue", "lastBlock", req.LastBlock, "done", req.Done) if err := u.SetRequest(id, &res); err != nil { log.Errorf("error updating request in the queue: %v", err) } + log.Infow("token processed", + "address", res.Address.Hex(), + "lastBlock", res.LastBlock, + "done", res.Done) }(id, *req) } } @@ -216,6 +216,11 @@ func RequestID(address common.Address, chainID uint64, externalID string) (strin return hex.EncodeToString(bHash[:4]), nil } +// next returns the next request in the queue that is not being processed or +// already done. It will return the request and its ID. If the queue is empty or +// the next request is out of the range of the sorted queue, it will return nil +// and an it will return nil and an empty string. If the next request is found +// it updates the next request index to the next request in the sorted queue. func (u *Updater) next() (*UpdateRequest, string) { u.queueMtx.Lock() defer u.queueMtx.Unlock()