reconfigure probes to unhealthy if defid blocks are more than 4 heade…

…rs behind (#331)

* reconfigure probes for idb

defid.readiness dead if defid blocks are more than 4 headers behind
model.readiness dead synced blocks are more than 4 blocks behind

* added probes healthy with peers testing

* Update _model.probes.ts

apps/whale-api/src/module.api/actuator.e2e.ts

@@ -1,29 +1,133 @@
-import { MasterNodeRegTestContainer } from '@defichain/testcontainers'
+import { MasternodeGroup, MasterNodeRegTestContainer } from '@defichain/testcontainers'
 import { NestFastifyApplication } from '@nestjs/platform-fastify'
 import { createTestingApp, waitForIndexedHeight } from '@src/e2e.module'
 
-const container = new MasterNodeRegTestContainer()
-let app: NestFastifyApplication
+describe('no peers', () => {
+  const container = new MasterNodeRegTestContainer()
+  let app: NestFastifyApplication
 
-beforeAll(async () => {
-  await container.start()
-  await container.waitForReady()
-  app = await createTestingApp(container)
+  beforeAll(async () => {
+    await container.start()
+    app = await createTestingApp(container)
 
-  await container.generate(3)
-  await waitForIndexedHeight(app, 2)
-})
+    await container.generate(3)
+    await waitForIndexedHeight(app, 2)
+  })
+
+  afterAll(async () => {
+    try {
+      await app.close()
+    } finally {
+      await container.stop()
+    }
+  })
+
+  describe('/_actuator/probes/liveness', () => {
+    it('should wait until liveness', async () => {
+      const res = await app.inject({
+        method: 'GET',
+        url: '/_actuator/probes/liveness'
+      })
 
-afterAll(async () => {
-  try {
-    await app.close()
-  } finally {
-    await container.stop()
-  }
+      expect(res.statusCode).toStrictEqual(200)
+      expect(res.json()).toStrictEqual({
+        details: {
+          defid: {
+            status: 'up'
+          },
+          model: {
+            status: 'up'
+          }
+        },
+        error: {},
+        info: {
+          defid: {
+            status: 'up'
+          },
+          model: {
+            status: 'up'
+          }
+        },
+        status: 'ok'
+      })
+    })
+  })
+
+  describe('/_actuator/probes/readiness', () => {
+    it('should wait until readiness, but never will be as it lacks connections', async () => {
+      const res = await app.inject({
+        method: 'GET',
+        url: '/_actuator/probes/readiness'
+      })
+
+      expect(res.statusCode).toStrictEqual(503)
+      expect(res.json()).toStrictEqual({
+        details: {
+          defid: {
+            blocks: expect.any(Number),
+            headers: expect.any(Number),
+            initialBlockDownload: expect.any(Boolean),
+            peers: 0,
+            status: 'down'
+          },
+          model: {
+            count: {
+              defid: expect.any(Number),
+              index: expect.any(Number)
+            },
+            status: 'up'
+          }
+        },
+        error: {
+          defid: {
+            blocks: expect.any(Number),
+            headers: expect.any(Number),
+            initialBlockDownload: expect.any(Boolean),
+            peers: 0,
+            status: 'down'
+          }
+        },
+        info: {
+          model: {
+            count: {
+              defid: expect.any(Number),
+              index: expect.any(Number)
+            },
+            status: 'up'
+          }
+        },
+        status: 'error'
+      })
+    })
+  })
 })
 
-describe('/_actuator/probes/liveness', () => {
-  it('should wait until liveness', async () => {
+describe('with peers', () => {
+  const group = new MasternodeGroup([
+    new MasterNodeRegTestContainer(),
+    new MasterNodeRegTestContainer()
+  ])
+  const container = group.get(0)
+  let app: NestFastifyApplication
+
+  beforeAll(async () => {
+    await group.start()
+    app = await createTestingApp(container)
+
+    await container.generate(10)
+    await group.waitForSync()
+    await waitForIndexedHeight(app, 9)
+  })
+
+  afterAll(async () => {
+    try {
+      await app.close()
+    } finally {
+      await group.stop()
+    }
+  })
+
+  it('/_actuator/probes/liveness healthy', async () => {
     const res = await app.inject({
       method: 'GET',
       url: '/_actuator/probes/liveness'
@@ -51,26 +155,22 @@ describe('/_actuator/probes/liveness', () => {
       status: 'ok'
     })
   })
-})
 
-describe('/_actuator/probes/readiness', () => {
-  // TODO(fuxingloh): /_actuator/probes/readiness tests for it to be ready
-
-  it('should wait until readiness, but never will be as it lacks connections', async () => {
+  it('/_actuator/probes/readiness healthy', async () => {
     const res = await app.inject({
       method: 'GET',
       url: '/_actuator/probes/readiness'
     })
 
-    expect(res.statusCode).toStrictEqual(503)
+    expect(res.statusCode).toStrictEqual(200)
     expect(res.json()).toStrictEqual({
       details: {
         defid: {
           blocks: expect.any(Number),
           headers: expect.any(Number),
-          initialBlockDownload: expect.any(Boolean),
-          peers: 0,
-          status: 'down'
+          initialBlockDownload: false,
+          peers: 1,
+          status: 'up'
         },
         model: {
           count: {
@@ -80,16 +180,15 @@ describe('/_actuator/probes/readiness', () => {
           status: 'up'
         }
       },
-      error: {
+      error: {},
+      info: {
         defid: {
           blocks: expect.any(Number),
           headers: expect.any(Number),
-          initialBlockDownload: expect.any(Boolean),
-          peers: 0,
-          status: 'down'
-        }
-      },
-      info: {
+          initialBlockDownload: false,
+          peers: 1,
+          status: 'up'
+        },
         model: {
           count: {
             defid: expect.any(Number),
@@ -98,7 +197,7 @@ describe('/_actuator/probes/readiness', () => {
           status: 'up'
         }
       },
-      status: 'error'
+      status: 'ok'
     })
   })
 })

apps/whale-api/src/module.defid/defid.probes.ts

@@ -45,8 +45,8 @@ export class DeFiDProbeIndicator extends ProbeIndicator {
       peers: peers
     }
 
-    if (info.initialblockdownload) {
-      return this.withDead('defid', 'defid in initial block download', details)
+    if (info.blocks + 4 <= info.headers) {
+      return this.withDead('defid', 'defid blocks are more than 4 headers behind', details)
     }
 
     if (peers === 0) {

apps/whale-api/src/module.model/_model.probes.ts

@@ -31,7 +31,7 @@ export class ModelProbeIndicator extends ProbeIndicator {
    * Readiness of Model Database
    * - unable to get the latest block
    * - synced blocks are undefined
-   * - synced blocks are more than 5 blocks behind
+   * - synced blocks are more than 4 blocks behind
    */
   async readiness (): Promise<HealthIndicatorResult> {
     let index: number | undefined
@@ -55,8 +55,8 @@ export class ModelProbeIndicator extends ProbeIndicator {
       return this.withDead('model', 'synced blocks are undefined', details)
     }
 
-    if (index + 5 <= defid) { // allowed to be up to 5 block behind
-      return this.withDead('model', 'synced blocks are more than 5 blocks behind', details)
+    if (index + 4 <= defid) {
+      return this.withDead('model', 'synced blocks are more than 4 blocks behind', details)
     }
 
     return this.withAlive('model', details)