/blocks & /transactions improvement (#385)

* Fixed tx ordering, added block reward and vout total

* Tests passing

* Refactor tests and rename vout

* PR feedback,. simplified db structure, fixed test

* Fix api test

* Correct test

* Fix test

* Apply suggestions from code review

Co-authored-by: Fuxing Loh <[email protected]>

apps/whale/src/module.api/block.controller.e2e.ts

@@ -2,10 +2,12 @@ import { BlockController, parseHeight } from '@src/module.api/block.controller'
 import { MasterNodeRegTestContainer } from '@defichain/testcontainers'
 import { NestFastifyApplication } from '@nestjs/platform-fastify'
 import { createTestingApp, stopTestingApp, waitForIndexedHeight } from '@src/e2e.module'
+import { JsonRpcClient } from '@defichain/jellyfish-api-jsonrpc'
 
 const container = new MasterNodeRegTestContainer()
 let app: NestFastifyApplication
 let controller: BlockController
+let client: JsonRpcClient
 
 beforeAll(async () => {
   await container.start()
@@ -15,6 +17,15 @@ beforeAll(async () => {
 
   await waitForIndexedHeight(app, 100)
   controller = app.get(BlockController)
+  client = new JsonRpcClient(await container.getCachedRpcUrl())
+
+  const address = await container.getNewAddress()
+  for (let i = 0; i < 4; i += 1) {
+    await container.call('sendtoaddress', [address, 0.1])
+  }
+
+  await container.generate(3)
+  await waitForIndexedHeight(app, 103)
 })
 
 afterAll(async () => {
@@ -84,14 +95,14 @@ describe('list', () => {
 describe('getTransactions', () => {
   it('should get transactions from a block by hash', async () => {
     const blockHash = await container.call('getblockhash', [100])
-    const paginatedTransactions = await controller.getTransactions(blockHash, { size: 30, next: '10' })
+    const paginatedTransactions = await controller.getTransactions(blockHash, { size: 30 })
 
     expect(paginatedTransactions.data.length).toBeGreaterThanOrEqual(1)
     expect(paginatedTransactions.data[0].block.height).toStrictEqual(100)
   })
 
   it('getTransactions should not get transactions by height', async () => {
-    const paginatedTransactions = await controller.getTransactions('0', { size: 30, next: '10' })
+    const paginatedTransactions = await controller.getTransactions('0', { size: 30 })
 
     expect(paginatedTransactions.data.length).toStrictEqual(0)
   })
@@ -107,6 +118,20 @@ describe('getTransactions', () => {
 
     expect(paginatedTransactions.data.length).toStrictEqual(0)
   })
+
+  it('should list transactions in the right order', async () => {
+    const blockHash = await container.call('getblockhash', [103])
+    const paginatedTransactions = await controller.getTransactions(blockHash, { size: 30 })
+
+    expect(paginatedTransactions.data.length).toBeGreaterThanOrEqual(4)
+    expect(paginatedTransactions.data[0].block.height).toStrictEqual(103)
+
+    const rpcBlock = await client.blockchain.getBlock(blockHash, 2)
+    expect(paginatedTransactions.data[0].hash).toStrictEqual(rpcBlock.tx[0].hash)
+    expect(paginatedTransactions.data[1].hash).toStrictEqual(rpcBlock.tx[1].hash)
+    expect(paginatedTransactions.data[2].hash).toStrictEqual(rpcBlock.tx[2].hash)
+    expect(paginatedTransactions.data[3].hash).toStrictEqual(rpcBlock.tx[3].hash)
+  })
 })
 
 describe('parseHeight', () => {

apps/whale/src/module.api/block.controller.ts

@@ -1,5 +1,5 @@
 import { Controller, Get, Param, Query } from '@nestjs/common'
-import { Block, BlockMapper } from '@src/module.model/block'
+import { BlockMapper, Block } from '@src/module.model/block'
 import { Transaction, TransactionMapper } from '@src/module.model/transaction'
 import { ApiPagedResponse } from '@src/module.api/_core/api.paged.response'
 import { PaginationQuery } from '@src/module.api/_core/api.query'
@@ -51,9 +51,9 @@ export class BlockController {
       return ApiPagedResponse.empty()
     }
 
-    const transactions = await this.transactionMapper.queryByBlockHash(hash, query.size, query.next)
+    const transactions = await this.transactionMapper.queryByBlockHash(hash, query.size, parseHeight(query.next))
     return ApiPagedResponse.of(transactions, query.size, transaction => {
-      return transaction.id
+      return transaction.order.toString()
     })
   }
 }

apps/whale/src/module.api/transaction.controller.e2e.ts

@@ -59,6 +59,7 @@ describe('get', () => {
     const transaction = await controller.get(txid)
     expect(transaction).toStrictEqual({
       id: txid,
+      order: expect.any(Number),
       block: {
         hash: expect.any(String),
         height: expect.any(Number),
@@ -73,7 +74,8 @@ describe('get', () => {
       weight: expect.any(Number),
       lockTime: expect.any(Number),
       vinCount: expect.any(Number),
-      voutCount: expect.any(Number)
+      voutCount: expect.any(Number),
+      totalVoutValue: expect.any(String)
     })
   })
 

apps/whale/src/module.api/transaction.controller.ts

@@ -1,6 +1,5 @@
 import { Controller, Get, NotFoundException, Param, Query } from '@nestjs/common'
-import { TransactionMapper } from '@src/module.model/transaction'
-import { Transaction } from '@whale-api-client/api/transactions'
+import { TransactionMapper, Transaction } from '@src/module.model/transaction'
 import { PaginationQuery } from '@src/module.api/_core/api.query'
 import { ApiPagedResponse } from '@src/module.api/_core/api.paged.response'
 import { TransactionVin, TransactionVinMapper } from '@src/module.model/transaction.vin'

apps/whale/src/module.indexer/model/block.ts

@@ -9,14 +9,15 @@ export class BlockIndexer extends Indexer {
   }
 
   async index (block: RawBlock): Promise<void> {
-    await this.mapper.put(this.map(block))
+    const reward = block.tx[0].vout[0].value.toFixed(8)
+    await this.mapper.put(this.map(block, reward))
   }
 
   async invalidate (block: RawBlock): Promise<void> {
     await this.mapper.delete(block.hash)
   }
 
-  map (block: RawBlock): Block {
+  map (block: RawBlock, reward: string): Block {
     return {
       id: block.hash,
       hash: block.hash,
@@ -34,7 +35,8 @@ export class BlockIndexer extends Indexer {
       merkleroot: block.merkleroot,
       size: block.size,
       sizeStripped: block.strippedsize,
-      weight: block.weight
+      weight: block.weight,
+      reward: reward
     }
   }
 }

apps/whale/src/module.indexer/model/transaction.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common'
 import { defid, Indexer, RawBlock } from '@src/module.indexer/model/_abstract'
 import { Transaction, TransactionMapper } from '@src/module.model/transaction'
+import BigNumber from 'bignumber.js'
 
 @Injectable()
 export class TransactionIndexer extends Indexer {
@@ -9,8 +10,11 @@ export class TransactionIndexer extends Indexer {
   }
 
   async index (block: RawBlock): Promise<void> {
-    for (const txn of block.tx) {
-      await this.mapper.put(this.map(block, txn))
+    for (const [order, txn] of block.tx.entries()) {
+      const totalVOut = txn.vout.reduce<BigNumber>((prev, vout) => {
+        return prev.plus(vout.value)
+      }, new BigNumber(0))
+      await this.mapper.put(this.map(block, txn, order, totalVOut.toFixed(8)))
     }
   }
 
@@ -20,9 +24,10 @@ export class TransactionIndexer extends Indexer {
     }
   }
 
-  map (block: RawBlock, txn: defid.Transaction): Transaction {
+  map (block: RawBlock, txn: defid.Transaction, order: number, totalVOut: string): Transaction {
     return {
       id: txn.txid,
+      order: order,
       block: {
         hash: block.hash,
         height: block.height,
@@ -37,7 +42,8 @@ export class TransactionIndexer extends Indexer {
       weight: txn.weight,
       lockTime: txn.locktime,
       vinCount: txn.vin.length,
-      voutCount: txn.vout.length
+      voutCount: txn.vout.length,
+      totalVoutValue: totalVOut
     }
   }
 }

apps/whale/src/module.model/block.spec.ts

@@ -37,7 +37,8 @@ beforeEach(async () => {
       time: 0,
       transactionCount: 0,
       version: 0,
-      weight: 0
+      weight: 0,
+      reward: ''
     })
   }
 

apps/whale/src/module.model/block.ts

@@ -73,6 +73,7 @@ export interface Block extends Model {
   masternode: string
   minter: string
   minterBlockCount: number
+  reward: string // ------------| reward string as decimal
 
   stakeModifier: string
   merkleroot: string

apps/whale/src/module.model/transaction.ts

@@ -5,15 +5,15 @@ import { Database, SortOrder } from '@src/module.database/database'
 const TransactionMapping: ModelMapping<Transaction> = {
   type: 'transaction',
   index: {
-    block_txid: {
-      name: 'transaction_block_txid',
+    block_order: {
+      name: 'transaction_block_order',
       partition: {
         type: 'string',
         key: (b: Transaction) => b.block.hash
       },
       sort: {
-        type: 'string',
-        key: (b: Transaction) => b.txid
+        type: 'number',
+        key: (b: Transaction) => b.order
       }
     }
   }
@@ -24,8 +24,8 @@ export class TransactionMapper {
   public constructor (protected readonly database: Database) {
   }
 
-  async queryByBlockHash (hash: string, limit: number, gt?: string): Promise<Transaction[]> {
-    return await this.database.query(TransactionMapping.index.block_txid, {
+  async queryByBlockHash (hash: string, limit: number, gt?: number): Promise<Transaction[]> {
+    return await this.database.query(TransactionMapping.index.block_order, {
       partitionKey: hash,
       limit: limit,
       order: SortOrder.ASC,
@@ -51,6 +51,7 @@ export class TransactionMapper {
  */
 export interface Transaction extends Model {
   id: string // ----------------| unique id of the transaction, same as the txid
+  order: number // --------------| tx order
 
   block: {
     hash: string
@@ -66,6 +67,7 @@ export interface Transaction extends Model {
   size: number
   vSize: number
   weight: number
+  totalVoutValue: string
 
   lockTime: number