feat(server): Server should periodically delete old builds

packages/server/src/api/storage/sql/sql.js

@@ -456,6 +456,19 @@ class SqlStorageMethod {
     }
   }
 
+  /**
+   * @param {Date} runAt
+   * @return {Promise<LHCI.ServerCommand.Build[]>}
+   */
+  async findOldBuilds(runAt) {
+    const {buildModel} = this._sql();
+    const oldBuilds = await buildModel.findAll({
+      where: {runAt: {[Sequelize.Op.lte]: runAt}},
+      order: [['runAt', 'ASC']],
+    });
+    return oldBuilds;
+  }
+
   /**
    * @param {string} projectId
    * @param {string} buildId

packages/server/src/api/storage/storage-method.js

@@ -148,6 +148,15 @@ class StorageMethod {
     throw new Error('Unimplemented');
   }
 
+  /**
+   * @param {Date} runAt
+   * @return {Promise<LHCI.ServerCommand.Build[]>}
+   */
+  // eslint-disable-next-line no-unused-vars
+  async findOldBuilds(runAt) {
+    throw new Error('Unimplemented');
+  }
+
   /**
    * @param {string} projectId
    * @param {string} buildId

packages/server/src/cron/delete-old-builds.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const {CronJob} = require('cron');
+const {normalizeCronSchedule} = require('./utils');
+
+/**
+ * @param {LHCI.ServerCommand.StorageMethod} storageMethod
+ * @param {number} range
+ * @param {Date} now
+ * @return {Promise<void>}
+ */
+async function deleteOldBuilds(storageMethod, range, now = new Date()) {
+  if (!range || !Number.isInteger(range) || range <= 0) {
+    throw new Error('Invalid range');
+  }
+
+  const runAt = new Date(now.setDate(now.getDate() + range));
+  const oldBuilds = await storageMethod.findOldBuilds(runAt);
+  for (const {projectId, id} of oldBuilds) {
+    await storageMethod.deleteBuild(projectId, id);
+  }
+}
+
+/**
+ * @param {LHCI.ServerCommand.StorageMethod} storageMethod
+ * @param {LHCI.ServerCommand.Options} options
+ * @return {void}
+ */
+function startDeletingOldBuildsCron(storageMethod, options) {
+  if (options.storage.storageMethod !== 'sql' || !options.storage.deleteOldBuilds) {
+    return;
+  }
+
+  if (!options.storage.deleteOldBuilds.schedule || !options.storage.deleteOldBuilds.dateRange) {
+    throw new Error('Cannot configure schedule');
+  }
+
+  const log =
+    options.logLevel === 'silent'
+      ? () => {}
+      : msg => process.stdout.write(`${new Date().toISOString()} - ${msg}\n`);
+
+  let inProgress = false;
+
+  const {schedule, dateRange} = options.storage.deleteOldBuilds;
+
+  const cron = new CronJob(normalizeCronSchedule(schedule), () => {
+    if (inProgress) {
+      log(`Deleting old builds still in progress. Skipping...`);
+      return;
+    }
+    inProgress = true;
+    log(`Starting delete old builds`);
+    deleteOldBuilds(storageMethod, dateRange, new Date())
+      .then(() => {
+        log(`Successfully delete old builds`);
+      })
+      .catch(err => {
+        log(`Delete old builds failure: ${err.message}`);
+      })
+      .finally(() => {
+        inProgress = false;
+      });
+  });
+  cron.start();
+}
+module.exports = {startDeletingOldBuildsCron, deleteOldBuilds};

packages/server/src/cron/psi-collect.js

@@ -8,31 +8,7 @@
 const {CronJob} = require('cron');
 const {getGravatarUrlFromEmail} = require('@lhci/utils/src/build-context');
 const PsiRunner = require('@lhci/utils/src/psi-runner.js');
-
-/**
- * @param {string} schedule
- * @return {string}
- */
-function normalizeCronSchedule(schedule) {
-  if (typeof schedule !== 'string') {
-    throw new Error(`Schedule must be provided`);
-  }
-
-  if (process.env.OVERRIDE_SCHEDULE_FOR_TEST) {
-    return process.env.OVERRIDE_SCHEDULE_FOR_TEST;
-  }
-
-  const parts = schedule.split(/\s+/).filter(Boolean);
-  if (parts.length !== 5) {
-    throw new Error(`Invalid cron format, expected <minutes> <hours> <day> <month> <day of week>`);
-  }
-
-  if (parts[0] === '*') {
-    throw new Error(`Cron schedule "${schedule}" is too frequent`);
-  }
-
-  return ['0', ...parts].join(' ');
-}
+const {normalizeCronSchedule} = require('./utils');
 
 /**
  * @param {LHCI.ServerCommand.StorageMethod} storageMethod

packages/server/src/cron/utils.js

@@ -0,0 +1,26 @@
+'use strict';
+/**
+ * @param {string} schedule
+ * @return {string}
+ */
+function normalizeCronSchedule(schedule) {
+  if (typeof schedule !== 'string') {
+    throw new Error(`Schedule must be provided`);
+  }
+
+  if (process.env.OVERRIDE_SCHEDULE_FOR_TEST) {
+    return process.env.OVERRIDE_SCHEDULE_FOR_TEST;
+  }
+
+  const parts = schedule.split(/\s+/).filter(Boolean);
+  if (parts.length !== 5) {
+    throw new Error(`Invalid cron format, expected <minutes> <hours> <day> <month> <day of week>`);
+  }
+
+  if (parts[0] === '*') {
+    throw new Error(`Cron schedule "${schedule}" is too frequent`);
+  }
+
+  return ['0', ...parts].join(' ');
+}
+module.exports = {normalizeCronSchedule};

packages/server/src/server.js

@@ -18,6 +18,7 @@ const createProjectsRouter = require('./api/routes/projects.js');
 const StorageMethod = require('./api/storage/storage-method.js');
 const {errorMiddleware, createBasicAuthMiddleware} = require('./api/express-utils.js');
 const {startPsiCollectCron} = require('./cron/psi-collect.js');
+const {startDeletingOldBuildsCron} = require('./cron/delete-old-builds');
 const version = require('../package.json').version;
 
 const DIST_FOLDER = path.join(__dirname, '../dist');
@@ -55,6 +56,7 @@ async function createApp(options) {
   app.use(errorMiddleware);
 
   startPsiCollectCron(storageMethod, options);
+  startDeletingOldBuildsCron(storageMethod, options);
 
   return {app, storageMethod};
 }

packages/server/test/cron/delete-old-builds.test.js

@@ -0,0 +1,133 @@
+'use strict';
+
+/* eslint-env jest */
+
+/** @type {jest.MockInstance} */
+let cronJob = jest.fn().mockReturnValue({start: () => {}});
+jest.mock('cron', () => ({
+  CronJob: function(...args) {
+    // use this indirection because we have to invoke it with `new` and it's harder to mock assertions
+    return cronJob(...args);
+  },
+}));
+
+const {
+  startDeletingOldBuildsCron,
+  deleteOldBuilds,
+} = require('../../src/cron/delete-old-builds.js');
+
+describe('cron/delete-old-builds', () => {
+  /** @type {{ findOldBuilds: jest.MockInstance, deleteBuild: jest.MockInstance}} */
+  let storageMethod;
+  beforeEach(() => {
+    storageMethod = {
+      findOldBuilds: jest.fn().mockResolvedValue([]),
+      deleteBuild: jest.fn().mockResolvedValue({}),
+    };
+
+    cronJob = jest.fn().mockReturnValue({
+      start: () => {},
+    });
+  });
+  describe('.deleteOldBuilds', () => {
+    it.each([undefined, null, -1, new Date()])(
+      'should throw for invalid range (%s)',
+      async range => {
+        await expect(deleteOldBuilds(storageMethod, range)).rejects.toMatchObject({
+          message: 'Invalid range',
+        });
+      }
+    );
+
+    it('should collect', async () => {
+      storageMethod.deleteBuild.mockClear();
+      const deleteObjects = [{id: 'id-1', projectId: 'pid-1'}, {id: 'id-2', projectId: 'pid-2'}];
+      storageMethod.findOldBuilds.mockResolvedValue(deleteObjects);
+      await deleteOldBuilds(storageMethod, 30, new Date('2019-01-01'));
+      expect(storageMethod.deleteBuild).toHaveBeenCalledTimes(deleteObjects.length);
+
+      expect(storageMethod.deleteBuild.mock.calls).toMatchObject([
+        ['pid-1', 'id-1'],
+        ['pid-2', 'id-2'],
+      ]);
+    });
+  });
+
+  describe('.startDeletingOldBuildsCron', () => {
+    it.each([
+      [
+        'storageMethod is not sql',
+        {
+          storage: {
+            storageMethod: 'notsql',
+            deleteOldBuilds: {
+              schedule: '0 * * * *',
+              dateRange: 30,
+            },
+          },
+        },
+      ],
+      [
+        'no deleteOldBuilds options',
+        {
+          storage: {storageMethod: 'sql'},
+        },
+      ],
+    ])('should not schedule a cron job (%s)', (_, options) => {
+      startDeletingOldBuildsCron(storageMethod, options);
+      expect(cronJob).toHaveBeenCalledTimes(0);
+    });
+    it.each([
+      [
+        'no schedule',
+        {
+          storage: {
+            storageMethod: 'sql',
+            deleteOldBuilds: {
+              dateRange: 30,
+            },
+          },
+        },
+      ],
+      [
+        'no dateRagne',
+        {
+          storage: {
+            storageMethod: 'sql',
+            deleteOldBuilds: {
+              schedule: '0 * * * *',
+            },
+          },
+        },
+      ],
+    ])('should throw for invalid options (%s)', (_, options) => {
+      expect(() => startDeletingOldBuildsCron(storageMethod, options)).toThrow(/Cannot configure/);
+    });
+    it('should throw for invalid schedule', () => {
+      const options = {
+        storage: {
+          storageMethod: 'sql',
+          deleteOldBuilds: {
+            schedule: '* * *',
+            dateRange: 30,
+          },
+        },
+      };
+      expect(() => startDeletingOldBuildsCron(storageMethod, options)).toThrow(
+        /Invalid cron format/
+      );
+    });
+    it('should schedule a cron job', () => {
+      startDeletingOldBuildsCron(storageMethod, {
+        storage: {
+          storageMethod: 'sql',
+          deleteOldBuilds: {
+            schedule: '0 * * * *',
+            dateRange: 30,
+          },
+        },
+      });
+      expect(cronJob).toHaveBeenCalledTimes(1);
+    });
+  });
+});

packages/server/test/cron/utils.test.js

@@ -0,0 +1,21 @@
+'use strict';
+
+/* eslint-env jest */
+
+const {normalizeCronSchedule} = require('../../src/cron/utils.js');
+
+describe('cron/utils', () => {
+  describe('.normalizeCronSchedule()', () => {
+    it('should validate string', () => {
+      expect(() => normalizeCronSchedule(1)).toThrow(/Schedule must be provided/);
+    });
+
+    it('should validate cron job', () => {
+      expect(() => normalizeCronSchedule('* * * * *')).toThrow(/too frequent/);
+    });
+
+    it('should validate invalid format', () => {
+      expect(() => normalizeCronSchedule('* * *')).toThrow(/Invalid cron format/);
+    });
+  });
+});

types/collect.d.ts

@@ -44,7 +44,6 @@ declare global {
 
       export interface Options {
         url?: string | string[];
-        autodiscoverUrlBlocklist?: string | string[];
         psiApiKey?: string;
         psiApiEndpoint?: string;
         staticDistDir?: string;

types/server.d.ts

@@ -173,6 +173,10 @@ declare global {
         sqlConnectionUrl?: string;
         sqlDangerouslyResetDatabase?: boolean;
         sequelizeOptions?: import('sequelize').Options;
+        deleteOldBuilds?: {
+          schedule: string;
+          dateRange: number;
+        }
       }
 
       export interface Options {