new module.model for indexed data (#41)

* added better Create Pull Request body

* tested hex_encoder

* added module

* rename HexEncoder method

* updated readme and module

* added block.ts with test

* added raw.block.ts with test

* added script.activity.ts with test

* added the rest without testing

* fixed ci.yml script

apps/whale-api/src/app.module.ts

@@ -6,6 +6,7 @@ import { ApiModule } from '@src/module.api'
 import { DatabaseModule } from '@src/module.database/module'
 import { DeFiDModule } from '@src/module.defid'
 import { HealthModule } from '@src/module.health'
+import { ModelModule } from '@src/module.model/_module'
 import { AppConfiguration } from '@src/app.configuration'
 
 @Module({})
@@ -20,6 +21,7 @@ export class AppModule {
         }),
         ScheduleModule.forRoot(),
         DatabaseModule.forRoot(provider),
+        ModelModule,
         DeFiDModule,
         HealthModule,
         ApiModule

apps/whale-api/src/module.model/_hex.encoder.spec.ts

@@ -0,0 +1,31 @@
+import { HexEncoder } from '@src/module.model/_hex.encoder'
+
+it('should encode script hex', () => {
+  const hex = HexEncoder.asSHA256('1600140e7c0ab18b305bc987a266dc06de26fcfab4b56a')
+  expect(hex).toBe('3d78c27dffed5c633ec8cb3c1bab3aec7c63ff9247cd2a7646f98e4f7075cca0')
+  expect(hex.length).toBe(64)
+})
+
+it('should encode height', () => {
+  expect(HexEncoder.encodeHeight(0)).toBe('00000000')
+  expect(HexEncoder.encodeHeight(1)).toBe('00000001')
+  expect(HexEncoder.encodeHeight(255)).toBe('000000ff')
+  expect(HexEncoder.encodeHeight(256)).toBe('00000100')
+  expect(HexEncoder.encodeHeight(4294967295)).toBe('ffffffff')
+
+  expect(() => {
+    HexEncoder.encodeHeight(4294967296)
+  }).toThrow('max 32 bits but number larger than 4294967295')
+})
+
+it('should encode vout index', () => {
+  expect(HexEncoder.encodeVoutIndex(0)).toBe('00000000')
+  expect(HexEncoder.encodeVoutIndex(1)).toBe('00000001')
+  expect(HexEncoder.encodeVoutIndex(255)).toBe('000000ff')
+  expect(HexEncoder.encodeVoutIndex(256)).toBe('00000100')
+  expect(HexEncoder.encodeVoutIndex(4294967295)).toBe('ffffffff')
+
+  expect(() => {
+    HexEncoder.encodeVoutIndex(4294967296)
+  }).toThrow('max 32 bits but number larger than 4294967295')
+})

apps/whale-api/src/module.model/_hex.encoder.ts

@@ -0,0 +1,39 @@
+import { SHA256 } from '@defichain/jellyfish-crypto'
+
+/**
+ * HexEncoder to encode various DeFi transaction data into fixed length hex
+ * This allow the use of fixed length sorting in LSM database.
+ */
+export const HexEncoder = {
+  /**
+   * @param {string} hex to encode into HID 32 bytes hex-encoded string
+   * @return {string} fixed length of 32 byte
+   */
+  asSHA256 (hex: string): string {
+    const buffer = Buffer.from(hex, 'hex')
+    return SHA256(buffer).toString('hex')
+  },
+
+  /**
+   * 4 byte hex, Max Number = 4294967295
+   * @param {number} height from block to hex, about 4000 years
+   * @return {string} fixed length of 4 byte
+   */
+  encodeHeight (height: number): string {
+    if (height > 4294967295) {
+      throw new Error('max 32 bits but number larger than 4294967295')
+    }
+    return height.toString(16).padStart(8, '0')
+  },
+  /**
+   * 4 byte hex, Max Number = 4294967295
+   * @param {number} n from vout to hex, 4 bytes max consensus rule
+   * @return {string} fixed length of 4 byte
+   */
+  encodeVoutIndex (n: number): string {
+    if (n > 4294967295) {
+      throw new Error('max 32 bits but number larger than 4294967295')
+    }
+    return n.toString(16).padStart(8, '0')
+  }
+}

apps/whale-api/src/module.model/_module.ts

@@ -0,0 +1,25 @@
+import { Global, Module } from '@nestjs/common'
+import { RawBlockMapper } from '@src/module.model/raw.block'
+import { BlockMapper } from '@src/module.model/block'
+import { ScriptActivityMapper } from '@src/module.model/script.activity'
+import { ScriptAggregationMapper } from '@src/module.model/script.aggregation'
+import { ScriptUnspentMapper } from '@src/module.model/script.unspent'
+import { TransactionMapper } from '@src/module.model/transaction'
+import { TransactionVinMapper } from '@src/module.model/transaction.vin'
+import { TransactionVoutMapper } from '@src/module.model/transaction.vout'
+
+@Global()
+@Module({
+  providers: [
+    RawBlockMapper,
+    BlockMapper,
+    ScriptActivityMapper,
+    ScriptAggregationMapper,
+    ScriptUnspentMapper,
+    TransactionMapper,
+    TransactionVinMapper,
+    TransactionVoutMapper
+  ]
+})
+export class ModelModule {
+}

apps/whale-api/src/module.model/block.spec.ts

@@ -0,0 +1,114 @@
+import { Database } from '@src/module.database/database'
+import { Test } from '@nestjs/testing'
+import { MemoryDatabaseModule } from '@src/module.database/provider.memory/module'
+import { LevelDatabase } from '@src/module.database/provider.level/level.database'
+import { BlockMapper } from '@src/module.model/block'
+import assert from 'assert'
+
+let database: Database
+let mapper: BlockMapper
+
+beforeAll(async () => {
+  const app = await Test.createTestingModule({
+    imports: [MemoryDatabaseModule],
+    providers: [BlockMapper]
+  }).compile()
+
+  database = app.get<Database>(Database)
+  mapper = app.get<BlockMapper>(BlockMapper)
+})
+
+afterAll(async () => {
+  await (database as LevelDatabase).close()
+})
+
+beforeEach(async () => {
+  async function put (height: number, hash: string): Promise<void> {
+    await mapper.put({
+      difficulty: 0,
+      id: hash,
+      hash: hash,
+      height: height,
+      masternode: '',
+      median_time: 0,
+      merkleroot: '',
+      minter: '',
+      minter_block_count: 0,
+      previous_hash: '',
+      size: 0,
+      size_stripped: 0,
+      stake_modifier: '',
+      time: 0,
+      transaction_count: 0,
+      version: 0,
+      weight: 0
+    })
+  }
+
+  await put(0, '0000000000000000000000000000000000000000000000000000000000000000')
+  await put(1, '1000000000000000000000000000000000000000000000000000000000000000')
+  await put(2, '1000000000000000000000000000000010000000000000000000000000000000')
+})
+
+afterEach(async () => {
+  await (database as LevelDatabase).clear()
+})
+
+it('should getByHash', async () => {
+  const block = await mapper.getByHash('1000000000000000000000000000000000000000000000000000000000000000')
+  expect(block?.height).toBe(1)
+})
+
+it('should getByHeight', async () => {
+  const block = await mapper.getByHeight(0)
+  expect(block?.hash).toBe('0000000000000000000000000000000000000000000000000000000000000000')
+})
+
+it('should getHighest', async () => {
+  const block = await mapper.getHighest()
+  expect(block?.height).toBe(2)
+  expect(block?.hash).toBe('1000000000000000000000000000000010000000000000000000000000000000')
+})
+
+it('should queryByHeight', async () => {
+  const blocks = await mapper.queryByHeight(10)
+  expect(blocks.length).toBe(3)
+
+  expect(blocks[0].height).toBe(2)
+  expect(blocks[1].height).toBe(1)
+  expect(blocks[2].height).toBe(0)
+
+  const after2 = await mapper.queryByHeight(10, 2)
+  expect(after2[0].height).toBe(1)
+  expect(after2[1].height).toBe(0)
+})
+
+it('should put', async () => {
+  const block = await mapper.getByHeight(0)
+  assert(block !== undefined)
+  block.size = 100
+  await mapper.put(block)
+
+  const updated = await mapper.getByHeight(0)
+  expect(updated?.size).toBe(100)
+})
+
+it('should put but deleted', async () => {
+  const block = await mapper.getByHeight(0)
+  assert(block !== undefined)
+
+  block.height = 3
+  await mapper.put(block)
+
+  const deleted = await mapper.getByHeight(0)
+  expect(deleted).toBeUndefined()
+
+  const updated = await mapper.getByHeight(3)
+  expect(updated).toBeTruthy()
+})
+
+it('should delete', async () => {
+  await mapper.delete('0000000000000000000000000000000000000000000000000000000000000000')
+  const deleted = await mapper.getByHeight(0)
+  expect(deleted).toBeUndefined()
+})

apps/whale-api/src/module.models/block.ts → apps/whale-api/src/module.model/block.ts

@@ -5,13 +5,6 @@ import { Database, SortOrder } from '@src/module.database/database'
 const BlockMapping: ModelMapping<Block> = {
   type: 'block',
   index: {
-    hash: {
-      name: 'block_hash',
-      partition: {
-        type: 'string',
-        key: (d: Block) => d.hash
-      }
-    },
     height: {
       name: 'block_height',
       partition: {
@@ -23,19 +16,19 @@ const BlockMapping: ModelMapping<Block> = {
 }
 
 @Injectable()
-export class BlockDbMapper {
+export class BlockMapper {
   public constructor (protected readonly database: Database) {
   }
 
   async getByHash (hash: string): Promise<Block | undefined> {
-    return await this.database.get(BlockMapping.index.hash, hash)
+    return await this.database.get(BlockMapping, hash)
   }
 
   async getByHeight (height: number): Promise<Block | undefined> {
     return await this.database.get(BlockMapping.index.height, height)
   }
 
-  async getBest (): Promise<Block | undefined> {
+  async getHighest (): Promise<Block | undefined> {
     const blocks = await this.database.query(BlockMapping.index.height, {
       order: SortOrder.DESC,
       limit: 1

apps/whale-api/src/module.model/raw.block.spec.ts

@@ -0,0 +1,51 @@
+import { MasterNodeRegTestContainer } from '@defichain/testcontainers'
+import { Database } from '@src/module.database/database'
+import { Test } from '@nestjs/testing'
+import { MemoryDatabaseModule } from '@src/module.database/provider.memory/module'
+import { LevelDatabase } from '@src/module.database/provider.level/level.database'
+import { RawBlockMapper } from '@src/module.model/raw.block'
+import { JsonRpcClient } from '@defichain/jellyfish-api-jsonrpc'
+
+const container = new MasterNodeRegTestContainer()
+let client: JsonRpcClient
+let database: Database
+let mapper: RawBlockMapper
+
+beforeAll(async () => {
+  await container.start()
+  await container.waitForReady()
+  await container.generate(3)
+  client = new JsonRpcClient(await container.getCachedRpcUrl())
+
+  const app = await Test.createTestingModule({
+    imports: [MemoryDatabaseModule],
+    providers: [RawBlockMapper]
+  }).compile()
+
+  database = app.get<Database>(Database)
+  mapper = app.get<RawBlockMapper>(RawBlockMapper)
+})
+
+afterAll(async () => {
+  await (database as LevelDatabase).close()
+  await container.stop()
+})
+
+it('should put from defid and get back same object from mapper', async () => {
+  const hash = await client.blockchain.getBlockHash(1)
+  const block = await client.blockchain.getBlock(hash, 2)
+  await mapper.put(block)
+
+  const saved = await mapper.get(hash)
+  expect(saved).toEqual(block)
+})
+
+it('should delete and be deleted', async () => {
+  const hash = await client.blockchain.getBlockHash(2)
+  const block = await client.blockchain.getBlock(hash, 2)
+  await mapper.put(block)
+  await mapper.delete(hash)
+
+  const deleted = await mapper.get(hash)
+  expect(deleted).toBeUndefined()
+})

apps/whale-api/src/module.model/raw.block.ts

@@ -0,0 +1,51 @@
+import { Injectable } from '@nestjs/common'
+import { Model, ModelMapping } from '@src/module.database/model'
+import { Database } from '@src/module.database/database'
+import { blockchain as defid } from '@defichain/jellyfish-api-core'
+import { JellyfishJSON } from '@defichain/jellyfish-json'
+
+const RawBlockMapping: ModelMapping<RawBlock> = {
+  type: 'raw_block',
+  index: {}
+}
+
+/**
+ * RawBlock data from defid with verbose 2, Block<Transaction>
+ * Data indexed in store for module.sync to reorg index in event of chain split.
+ */
+@Injectable()
+export class RawBlockMapper {
+  public constructor (protected readonly database: Database) {
+  }
+
+  async get (hash: string): Promise<defid.Block<defid.Transaction> | undefined> {
+    const cached = await this.database.get(RawBlockMapping, hash)
+    if (cached === undefined) {
+      return undefined
+    }
+
+    return JellyfishJSON.parse(cached.json, {
+      tx: {
+        vout: {
+          value: 'bignumber'
+        }
+      }
+    })
+  }
+
+  async put (block: defid.Block<defid.Transaction>): Promise<void> {
+    return await this.database.put(RawBlockMapping, {
+      id: block.hash,
+      json: JellyfishJSON.stringify(block)
+    })
+  }
+
+  async delete (hash: string): Promise<void> {
+    return await this.database.delete(RawBlockMapping, hash)
+  }
+}
+
+export interface RawBlock extends Model {
+  id: string // -----------------| hash
+  json: string // ---------------| block encoded in JSON with JellyfishJSON from defid with verbose 2
+}