Merge pull request #350 from hms-dbmi-cellenics/kubernetes-v2

added kubernetes.v2 endpoint

src/api.v2/routes/kubernetes.js

@@ -0,0 +1,34 @@
+const AWSXRay = require('aws-xray-sdk');
+const k8s = require('@kubernetes/client-node');
+const getLogger = require('../../utils/getLogger');
+const { OK } = require('../../utils/responses');
+
+const kc = new k8s.KubeConfig();
+kc.loadFromDefault();
+
+const logger = getLogger();
+
+module.exports = {
+  'kubernetes#event': async (req, res, next) => {
+    logger.log('received kubernetes event');
+    try {
+      const {
+        reason, message, type, involvedObject: { name, namespace },
+      } = req.body;
+      logger.log(`[${reason}] received ${type} kubernetes event: ${message} ${name} in ${namespace}`);
+
+      // remove only pods in your namespace and due to backoff errors
+      if ((namespace.match('^pipeline-.*') || namespace.match('^worker-.*'))
+       && reason === 'BackOff' && type !== 'Normal' && message.includes('restarting')) {
+        const k8sApi = kc.makeApiClient(k8s.CoreV1Api);
+        logger.log(`removing pod ${name} in ${namespace}`);
+        await k8sApi.deleteNamespacedPod(name, namespace);
+      }
+      res.json(OK());
+    } catch (e) {
+      logger.error('error processing k8s event', e);
+      AWSXRay.getSegment().addError(e);
+      next(e);
+    }
+  },
+};

src/specs/api.v2.yaml

@@ -158,26 +158,26 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/HTTPError'        
+                $ref: '#/components/schemas/HTTPError'
         '401':
           description: The request lacks authentication credentials.
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/HTTPError'   
+                $ref: '#/components/schemas/HTTPError'
         '403':
           description: Forbidden request for this user.
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/HTTPError'     
+                $ref: '#/components/schemas/HTTPError'
         '404':
           description: Not found error.
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPError'
-        
+
     put:
       summary: Update processing configuration for an experiment
       description: Update processing configuration for an experiment
@@ -208,14 +208,14 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/HTTPError'                
+                $ref: '#/components/schemas/HTTPError'
         '404':
           description: Not found error.
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPError'
-        
+
   '/experiments':
     get:
       summary: Get all experiments
@@ -610,7 +610,7 @@ paths:
             schema:
               type: object
               properties:
-                oldPosition: 
+                oldPosition:
                   type: integer
                 newPosition:
                   type: integer
@@ -1008,6 +1008,28 @@ paths:
               required:
                 - userEmail
         description: email of the user to remove from experiment
+  /kubernetesEvents:
+    post:
+      summary: Monitoring of kubernetes cluster events
+      description: Events from Kubernetes Event Exporter relayed to the API.
+      operationId: receiveKubernetesEvent
+      x-eov-operation-id: kubernetes#event
+      x-eov-operation-handler: routes/kubernetes
+      responses:
+        '200':
+          description: 'A JSON-parseable was received by the server, *irrespective of whether it was correct/acceptable or not*.'
+          content:
+            text/plain:
+              schema:
+                type: string
+                pattern: ok
+        '500':
+          description: The data sent by the server could not be parsed as JSON.
+          content:
+            text/plain:
+              schema:
+                type: string
+                pattern: nok
 components:
   schemas:
     CreateExperiment:

tests/api.v2/routes/kubernetes.test.js

@@ -0,0 +1,99 @@
+const k8s = require('@kubernetes/client-node');
+
+
+jest.mock('@kubernetes/client-node');
+
+
+const deleteNamespacedPod = jest.fn();
+const patchNamespacedPod = jest.fn();
+const listNamespacedPod = jest.fn();
+const mockApi = {
+  deleteNamespacedPod,
+  patchNamespacedPod,
+  listNamespacedPod,
+};
+
+k8s.KubeConfig.mockImplementation(() => ({
+  loadFromDefault: jest.fn(),
+  makeApiClient: (() => mockApi),
+}));
+
+
+const removeRequest = {
+  metadata: {
+    name: 'pipeline-6f87dbcb55-wzvnk.16b31901b952f1ec',
+    namespace: 'pipeline-default',
+    uid: 'cf831e5f-29bf-4750-ae9a-8bbe4d1f93fd',
+    resourceVersion: '226149137',
+    creationTimestamp: '2021-10-31T11:09:44Z',
+    managedFields: [[Object]],
+  },
+  reason: 'BackOff',
+  message: 'Back-off restarting failed container',
+  source: {
+    component: 'kubelet',
+    host: 'fargate-ip-192-168-180-35.eu-west-1.compute.internal',
+  },
+  firstTimestamp: '2021-10-31T11:09:44Z',
+  lastTimestamp: '2021-10-31T11:13:28Z',
+  count: 16,
+  type: 'Warning',
+  eventTime: null,
+  reportingComponent: '',
+  reportingInstance: '',
+  involvedObject: {
+    kind: 'Pod',
+    namespace: 'pipeline-default',
+    name: 'pipeline-6f87dbcb55-wzvnk',
+    uid: 'd204be1b-6626-4320-8bae-3b203aff9562',
+    apiVersion: 'v1',
+    resourceVersion: '226145375',
+    fieldPath: 'spec.containers{pipeline}',
+    labels: {
+      activityId: 'wrong',
+      'eks.amazonaws.com/fargate-profile': 'pipeline-default',
+      'pod-template-hash': '6f87dbcb55',
+      sandboxId: 'default',
+      type: 'pipeline',
+    },
+    annotations: {
+      CapacityProvisioned: '1vCPU 5GB',
+      Logging: 'LoggingDisabled: LOGGING_CONFIGMAP_NOT_FOUND',
+    },
+  },
+};
+
+const express = require('express');
+const request = require('supertest');
+const expressLoader = require('../../../src/loaders/express');
+
+describe('tests for kubernetes route', () => {
+  let app = null;
+
+  beforeEach(async () => {
+    const mockApp = await expressLoader(express());
+    app = mockApp.app;
+  });
+
+  it('sending a remove request works', async (done) => {
+    request(app)
+      .post('/v2/kubernetesEvents')
+      .send(removeRequest)
+      .expect(200)
+      .end((err) => {
+        if (err) {
+          return done(err);
+        }
+
+        return done();
+      });
+  });
+
+  it('sending an empty request works', async (done) => {
+    request(app)
+      .post('/v2/kubernetesEvents')
+      .send(undefined)
+      .expect(500)
+      .end((err) => done(err));
+  });
+});

tests/api/routes/kubernetes.test.js

@@ -67,22 +67,14 @@ const express = require('express');
 const request = require('supertest');
 const expressLoader = require('../../../src/loaders/express');
 
-describe('tests for experiment route', () => {
+describe('tests for kubernetes route', () => {
   let app = null;
 
   beforeEach(async () => {
     const mockApp = await expressLoader(express());
     app = mockApp.app;
   });
 
-  afterEach(() => {
-    /**
-     * Most important since b'coz of caching, the mocked implementations sometimes does not reset
-     */
-    jest.resetModules();
-    jest.restoreAllMocks();
-  });
-
   it('sending a remove request works', async (done) => {
     request(app)
       .post('/v1/kubernetesEvents')