From 7b51886701f5f48a65e34d6574463aebcbff273d Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Mon, 9 May 2022 12:25:27 +0200 Subject: [PATCH 1/4] added kubernetes.v2 endpoint --- src/api.v2/routes/kubernetes.js | 33 ++++++++ src/specs/api.v2.yaml | 39 +++++++-- tests/api.v2/routes/kubernetes.test.js | 107 +++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 8 deletions(-) create mode 100644 src/api.v2/routes/kubernetes.js create mode 100644 tests/api.v2/routes/kubernetes.test.js diff --git a/src/api.v2/routes/kubernetes.js b/src/api.v2/routes/kubernetes.js new file mode 100644 index 000000000..04035448f --- /dev/null +++ b/src/api.v2/routes/kubernetes.js @@ -0,0 +1,33 @@ +const AWSXRay = require('aws-xray-sdk'); +const k8s = require('@kubernetes/client-node'); +const getLogger = require('../../utils/getLogger'); + +const kc = new k8s.KubeConfig(); +kc.loadFromDefault(); + +const logger = getLogger(); + +module.exports = { + 'kubernetes#event': async (req, res, next) => { + logger.log('received kubernetes event'); + try { + const { + reason, message, type, involvedObject: { name, namespace }, + } = req.body; + logger.log(`[${reason}] received ${type} kubernetes event: ${message} ${name} in ${namespace}`); + + // remove only pods in your namespace and due to backoff errors + if ((namespace.match('^pipeline-.*') || namespace.match('^worker-.*')) + && reason === 'BackOff' && type !== 'Normal' && message.includes('restarting')) { + const k8sApi = kc.makeApiClient(k8s.CoreV1Api); + logger.log(`removing pod ${name} in ${namespace}`); + await k8sApi.deleteNamespacedPod(name, namespace); + } + res.status(200).send('ok'); + } catch (e) { + logger.error('error processing k8s event', e); + AWSXRay.getSegment().addError(e); + next(e); + } + }, +}; diff --git a/src/specs/api.v2.yaml b/src/specs/api.v2.yaml index 79fdd0cfb..8b456d132 100644 --- a/src/specs/api.v2.yaml +++ b/src/specs/api.v2.yaml @@ -80,26 +80,26 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/HTTPError' + $ref: '#/components/schemas/HTTPError' '401': description: The request lacks authentication credentials. content: application/json: schema: - $ref: '#/components/schemas/HTTPError' + $ref: '#/components/schemas/HTTPError' '403': description: Forbidden request for this user. content: application/json: schema: - $ref: '#/components/schemas/HTTPError' + $ref: '#/components/schemas/HTTPError' '404': description: Not found error. content: application/json: schema: $ref: '#/components/schemas/HTTPError' - + put: summary: Update processing configuration for an experiment description: Update processing configuration for an experiment @@ -130,14 +130,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/HTTPError' + $ref: '#/components/schemas/HTTPError' '404': description: Not found error. content: application/json: schema: $ref: '#/components/schemas/HTTPError' - + '/experiments': get: summary: Get all experiments @@ -532,7 +532,7 @@ paths: schema: type: object properties: - oldPosition: + oldPosition: type: integer newPosition: type: integer @@ -828,7 +828,7 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPError' - + '/access/{experimentId}': get: summary: Get the users with access to an experiment @@ -861,6 +861,29 @@ paths: application/json: schema: $ref: ./models/HTTPError.v1.yaml + /kubernetesEvents: + post: + summary: Monitoring of kubernetes cluster events + description: Events from Kubernetes Event Exporter relayed to the API. + operationId: receiveKubernetesEvent + x-eov-operation-id: kubernetes#event + x-eov-operation-handler: routes/kubernetes + responses: + '200': + description: 'A JSON-parseable was received by the server, *irrespective of whether it was correct/acceptable or not*.' + content: + text/plain: + schema: + type: string + pattern: ok + '500': + description: The data sent by the server could not be parsed as JSON. + content: + text/plain: + schema: + type: string + pattern: nok + components: schemas: CreateExperiment: diff --git a/tests/api.v2/routes/kubernetes.test.js b/tests/api.v2/routes/kubernetes.test.js new file mode 100644 index 000000000..fe4e9bb3a --- /dev/null +++ b/tests/api.v2/routes/kubernetes.test.js @@ -0,0 +1,107 @@ +const k8s = require('@kubernetes/client-node'); + + +jest.mock('@kubernetes/client-node'); + + +const deleteNamespacedPod = jest.fn(); +const patchNamespacedPod = jest.fn(); +const listNamespacedPod = jest.fn(); +const mockApi = { + deleteNamespacedPod, + patchNamespacedPod, + listNamespacedPod, +}; + +k8s.KubeConfig.mockImplementation(() => ({ + loadFromDefault: jest.fn(), + makeApiClient: (() => mockApi), +})); + + +const removeRequest = { + metadata: { + name: 'pipeline-6f87dbcb55-wzvnk.16b31901b952f1ec', + namespace: 'pipeline-default', + uid: 'cf831e5f-29bf-4750-ae9a-8bbe4d1f93fd', + resourceVersion: '226149137', + creationTimestamp: '2021-10-31T11:09:44Z', + managedFields: [[Object]], + }, + reason: 'BackOff', + message: 'Back-off restarting failed container', + source: { + component: 'kubelet', + host: 'fargate-ip-192-168-180-35.eu-west-1.compute.internal', + }, + firstTimestamp: '2021-10-31T11:09:44Z', + lastTimestamp: '2021-10-31T11:13:28Z', + count: 16, + type: 'Warning', + eventTime: null, + reportingComponent: '', + reportingInstance: '', + involvedObject: { + kind: 'Pod', + namespace: 'pipeline-default', + name: 'pipeline-6f87dbcb55-wzvnk', + uid: 'd204be1b-6626-4320-8bae-3b203aff9562', + apiVersion: 'v1', + resourceVersion: '226145375', + fieldPath: 'spec.containers{pipeline}', + labels: { + activityId: 'wrong', + 'eks.amazonaws.com/fargate-profile': 'pipeline-default', + 'pod-template-hash': '6f87dbcb55', + sandboxId: 'default', + type: 'pipeline', + }, + annotations: { + CapacityProvisioned: '1vCPU 5GB', + Logging: 'LoggingDisabled: LOGGING_CONFIGMAP_NOT_FOUND', + }, + }, +}; + +const express = require('express'); +const request = require('supertest'); +const expressLoader = require('../../../src/loaders/express'); + +describe('tests for experiment route', () => { + let app = null; + + beforeEach(async () => { + const mockApp = await expressLoader(express()); + app = mockApp.app; + }); + + afterEach(() => { + /** + * Most important since b'coz of caching, the mocked implementations sometimes does not reset + */ + jest.resetModules(); + jest.restoreAllMocks(); + }); + + it('sending a remove request works', async (done) => { + request(app) + .post('/v2/kubernetesEvents') + .send(removeRequest) + .expect(200) + .end((err) => { + if (err) { + return done(err); + } + + return done(); + }); + }); + + it('sending an empty request works', async (done) => { + request(app) + .post('/v2/kubernetesEvents') + .send(undefined) + .expect(500) + .end((err) => done(err)); + }); +}); From 7b3c660e1092ed3d778b97627c69f887a8de67a4 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Mon, 9 May 2022 16:37:55 +0200 Subject: [PATCH 2/4] fixed wrong test name --- tests/api.v2/routes/kubernetes.test.js | 2 +- tests/api/routes/kubernetes.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/api.v2/routes/kubernetes.test.js b/tests/api.v2/routes/kubernetes.test.js index fe4e9bb3a..9788321c3 100644 --- a/tests/api.v2/routes/kubernetes.test.js +++ b/tests/api.v2/routes/kubernetes.test.js @@ -67,7 +67,7 @@ const express = require('express'); const request = require('supertest'); const expressLoader = require('../../../src/loaders/express'); -describe('tests for experiment route', () => { +describe('tests for kubernetes route', () => { let app = null; beforeEach(async () => { diff --git a/tests/api/routes/kubernetes.test.js b/tests/api/routes/kubernetes.test.js index 29ffeb7ce..6e3ccd289 100644 --- a/tests/api/routes/kubernetes.test.js +++ b/tests/api/routes/kubernetes.test.js @@ -67,7 +67,7 @@ const express = require('express'); const request = require('supertest'); const expressLoader = require('../../../src/loaders/express'); -describe('tests for experiment route', () => { +describe('tests for kubernetes route', () => { let app = null; beforeEach(async () => { From aa7657c2cca78029d8a270e025d8aa3f6303a513 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Mon, 9 May 2022 16:40:32 +0200 Subject: [PATCH 3/4] removed unneeded code from tests --- tests/api.v2/routes/kubernetes.test.js | 8 -------- tests/api/routes/kubernetes.test.js | 8 -------- 2 files changed, 16 deletions(-) diff --git a/tests/api.v2/routes/kubernetes.test.js b/tests/api.v2/routes/kubernetes.test.js index 9788321c3..b69e978d2 100644 --- a/tests/api.v2/routes/kubernetes.test.js +++ b/tests/api.v2/routes/kubernetes.test.js @@ -75,14 +75,6 @@ describe('tests for kubernetes route', () => { app = mockApp.app; }); - afterEach(() => { - /** - * Most important since b'coz of caching, the mocked implementations sometimes does not reset - */ - jest.resetModules(); - jest.restoreAllMocks(); - }); - it('sending a remove request works', async (done) => { request(app) .post('/v2/kubernetesEvents') diff --git a/tests/api/routes/kubernetes.test.js b/tests/api/routes/kubernetes.test.js index 6e3ccd289..1e2ae79cd 100644 --- a/tests/api/routes/kubernetes.test.js +++ b/tests/api/routes/kubernetes.test.js @@ -75,14 +75,6 @@ describe('tests for kubernetes route', () => { app = mockApp.app; }); - afterEach(() => { - /** - * Most important since b'coz of caching, the mocked implementations sometimes does not reset - */ - jest.resetModules(); - jest.restoreAllMocks(); - }); - it('sending a remove request works', async (done) => { request(app) .post('/v1/kubernetesEvents') From 43c4ad2519acefcf617b915124aca54e7f8f2252 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Tue, 10 May 2022 12:46:03 +0200 Subject: [PATCH 4/4] changed OK() --- src/api.v2/routes/kubernetes.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api.v2/routes/kubernetes.js b/src/api.v2/routes/kubernetes.js index 04035448f..6a7cdfe3b 100644 --- a/src/api.v2/routes/kubernetes.js +++ b/src/api.v2/routes/kubernetes.js @@ -1,6 +1,7 @@ const AWSXRay = require('aws-xray-sdk'); const k8s = require('@kubernetes/client-node'); const getLogger = require('../../utils/getLogger'); +const { OK } = require('../../utils/responses'); const kc = new k8s.KubeConfig(); kc.loadFromDefault(); @@ -23,7 +24,7 @@ module.exports = { logger.log(`removing pod ${name} in ${namespace}`); await k8sApi.deleteNamespacedPod(name, namespace); } - res.status(200).send('ok'); + res.json(OK()); } catch (e) { logger.error('error processing k8s event', e); AWSXRay.getSegment().addError(e);