From e0ce522864ad2f542c12748b3e87615532fb056f Mon Sep 17 00:00:00 2001 From: Fuxing Loh <4266087+fuxingloh@users.noreply.github.com> Date: Mon, 9 Aug 2021 17:54:12 +0800 Subject: [PATCH] reconfigure probes to unhealthy if defid blocks are more than 4 headers behind (#331) * reconfigure probes for idb defid.readiness dead if defid blocks are more than 4 headers behind model.readiness dead synced blocks are more than 4 blocks behind * added probes healthy with peers testing * Update _model.probes.ts --- apps/whale-api/src/module.api/actuator.e2e.ts | 169 ++++++++++++++---- .../src/module.defid/defid.probes.ts | 4 +- .../src/module.model/_model.probes.ts | 6 +- 3 files changed, 139 insertions(+), 40 deletions(-) diff --git a/apps/whale-api/src/module.api/actuator.e2e.ts b/apps/whale-api/src/module.api/actuator.e2e.ts index 4f0a83943b..a9a962973c 100644 --- a/apps/whale-api/src/module.api/actuator.e2e.ts +++ b/apps/whale-api/src/module.api/actuator.e2e.ts @@ -1,29 +1,133 @@ -import { MasterNodeRegTestContainer } from '@defichain/testcontainers' +import { MasternodeGroup, MasterNodeRegTestContainer } from '@defichain/testcontainers' import { NestFastifyApplication } from '@nestjs/platform-fastify' import { createTestingApp, waitForIndexedHeight } from '@src/e2e.module' -const container = new MasterNodeRegTestContainer() -let app: NestFastifyApplication +describe('no peers', () => { + const container = new MasterNodeRegTestContainer() + let app: NestFastifyApplication -beforeAll(async () => { - await container.start() - await container.waitForReady() - app = await createTestingApp(container) + beforeAll(async () => { + await container.start() + app = await createTestingApp(container) - await container.generate(3) - await waitForIndexedHeight(app, 2) -}) + await container.generate(3) + await waitForIndexedHeight(app, 2) + }) + + afterAll(async () => { + try { + await app.close() + } finally { + await container.stop() + } + }) + + describe('/_actuator/probes/liveness', () => { + it('should wait until liveness', async () => { + const res = await app.inject({ + method: 'GET', + url: '/_actuator/probes/liveness' + }) -afterAll(async () => { - try { - await app.close() - } finally { - await container.stop() - } + expect(res.statusCode).toStrictEqual(200) + expect(res.json()).toStrictEqual({ + details: { + defid: { + status: 'up' + }, + model: { + status: 'up' + } + }, + error: {}, + info: { + defid: { + status: 'up' + }, + model: { + status: 'up' + } + }, + status: 'ok' + }) + }) + }) + + describe('/_actuator/probes/readiness', () => { + it('should wait until readiness, but never will be as it lacks connections', async () => { + const res = await app.inject({ + method: 'GET', + url: '/_actuator/probes/readiness' + }) + + expect(res.statusCode).toStrictEqual(503) + expect(res.json()).toStrictEqual({ + details: { + defid: { + blocks: expect.any(Number), + headers: expect.any(Number), + initialBlockDownload: expect.any(Boolean), + peers: 0, + status: 'down' + }, + model: { + count: { + defid: expect.any(Number), + index: expect.any(Number) + }, + status: 'up' + } + }, + error: { + defid: { + blocks: expect.any(Number), + headers: expect.any(Number), + initialBlockDownload: expect.any(Boolean), + peers: 0, + status: 'down' + } + }, + info: { + model: { + count: { + defid: expect.any(Number), + index: expect.any(Number) + }, + status: 'up' + } + }, + status: 'error' + }) + }) + }) }) -describe('/_actuator/probes/liveness', () => { - it('should wait until liveness', async () => { +describe('with peers', () => { + const group = new MasternodeGroup([ + new MasterNodeRegTestContainer(), + new MasterNodeRegTestContainer() + ]) + const container = group.get(0) + let app: NestFastifyApplication + + beforeAll(async () => { + await group.start() + app = await createTestingApp(container) + + await container.generate(10) + await group.waitForSync() + await waitForIndexedHeight(app, 9) + }) + + afterAll(async () => { + try { + await app.close() + } finally { + await group.stop() + } + }) + + it('/_actuator/probes/liveness healthy', async () => { const res = await app.inject({ method: 'GET', url: '/_actuator/probes/liveness' @@ -51,26 +155,22 @@ describe('/_actuator/probes/liveness', () => { status: 'ok' }) }) -}) -describe('/_actuator/probes/readiness', () => { - // TODO(fuxingloh): /_actuator/probes/readiness tests for it to be ready - - it('should wait until readiness, but never will be as it lacks connections', async () => { + it('/_actuator/probes/readiness healthy', async () => { const res = await app.inject({ method: 'GET', url: '/_actuator/probes/readiness' }) - expect(res.statusCode).toStrictEqual(503) + expect(res.statusCode).toStrictEqual(200) expect(res.json()).toStrictEqual({ details: { defid: { blocks: expect.any(Number), headers: expect.any(Number), - initialBlockDownload: expect.any(Boolean), - peers: 0, - status: 'down' + initialBlockDownload: false, + peers: 1, + status: 'up' }, model: { count: { @@ -80,16 +180,15 @@ describe('/_actuator/probes/readiness', () => { status: 'up' } }, - error: { + error: {}, + info: { defid: { blocks: expect.any(Number), headers: expect.any(Number), - initialBlockDownload: expect.any(Boolean), - peers: 0, - status: 'down' - } - }, - info: { + initialBlockDownload: false, + peers: 1, + status: 'up' + }, model: { count: { defid: expect.any(Number), @@ -98,7 +197,7 @@ describe('/_actuator/probes/readiness', () => { status: 'up' } }, - status: 'error' + status: 'ok' }) }) }) diff --git a/apps/whale-api/src/module.defid/defid.probes.ts b/apps/whale-api/src/module.defid/defid.probes.ts index ded90a55ac..be98d4fa99 100644 --- a/apps/whale-api/src/module.defid/defid.probes.ts +++ b/apps/whale-api/src/module.defid/defid.probes.ts @@ -45,8 +45,8 @@ export class DeFiDProbeIndicator extends ProbeIndicator { peers: peers } - if (info.initialblockdownload) { - return this.withDead('defid', 'defid in initial block download', details) + if (info.blocks + 4 <= info.headers) { + return this.withDead('defid', 'defid blocks are more than 4 headers behind', details) } if (peers === 0) { diff --git a/apps/whale-api/src/module.model/_model.probes.ts b/apps/whale-api/src/module.model/_model.probes.ts index 3abe75afe3..024cd59809 100644 --- a/apps/whale-api/src/module.model/_model.probes.ts +++ b/apps/whale-api/src/module.model/_model.probes.ts @@ -31,7 +31,7 @@ export class ModelProbeIndicator extends ProbeIndicator { * Readiness of Model Database * - unable to get the latest block * - synced blocks are undefined - * - synced blocks are more than 5 blocks behind + * - synced blocks are more than 4 blocks behind */ async readiness (): Promise { let index: number | undefined @@ -55,8 +55,8 @@ export class ModelProbeIndicator extends ProbeIndicator { return this.withDead('model', 'synced blocks are undefined', details) } - if (index + 5 <= defid) { // allowed to be up to 5 block behind - return this.withDead('model', 'synced blocks are more than 5 blocks behind', details) + if (index + 4 <= defid) { + return this.withDead('model', 'synced blocks are more than 4 blocks behind', details) } return this.withAlive('model', details)