feat: SQLite-based package cache

lib/config/types.ts

@@ -117,6 +117,7 @@ export interface GlobalOnlyConfig {
   repositories?: RenovateRepository[];
   platform?: PlatformId;
   endpoint?: string;
+  useSqliteCache?: boolean;
 }
 
 // Config options used within the repository worker, but not user configurable

lib/util/cache/package/index.ts

@@ -2,6 +2,7 @@ import type { AllConfig } from '../../../config/types';
 import * as memCache from '../memory';
 import * as fileCache from './file';
 import * as redisCache from './redis';
+import { SqlitePackageCache } from './sqlite';
 import type { PackageCache } from './types';
 
 let cacheProxy: PackageCache | undefined;
@@ -60,6 +61,22 @@ export async function init(config: AllConfig): Promise<void> {
       get: redisCache.get,
       set: redisCache.set,
     };
+  } else if (config.cacheDir && config.useSqliteCache) {
+    const sqlite = await SqlitePackageCache.init(config.cacheDir);
+    cacheProxy = {
+      get: (namespace: string, key: string) =>
+        Promise.resolve(sqlite.get(namespace, key)),
+      set: (
+        namespace: string,
+        key: string,
+        value: unknown,
+        ttlMinutes: number,
+      ) => Promise.resolve(sqlite.set(namespace, key, value, ttlMinutes)),
+      cleanup: () => {
+        sqlite.close();
+        return Promise.resolve();
+      },
+    };
   } else if (config.cacheDir) {
     fileCache.init(config.cacheDir);
     cacheProxy = {

lib/util/cache/package/sqlite.spec.ts

@@ -0,0 +1,33 @@
+import { withDir } from 'tmp-promise';
+import { SqlitePackageCache } from './sqlite';
+
+function withSqlite<T>(
+  fn: (sqlite: SqlitePackageCache) => T | Promise<T>,
+): Promise<T> {
+  return withDir(
+    async ({ path }) => {
+      const sqlite = await SqlitePackageCache.init(path);
+      const res = await fn(sqlite);
+      sqlite.close();
+      return res;
+    },
+    { unsafeCleanup: true },
+  );
+}
+
+describe('util/cache/package/sqlite', () => {
+  it('should get undefined', async () => {
+    const res = await withSqlite((sqlite) => sqlite.get('foo', 'bar'));
+    expect(res).toBeUndefined();
+  });
+
+  it('should set and get', async () => {
+    const res = await withSqlite((sqlite) => {
+      sqlite.set('foo', 'bar', { foo: 'foo' });
+      sqlite.set('foo', 'bar', { bar: 'bar' });
+      sqlite.set('foo', 'bar', { baz: 'baz' });
+      return sqlite.get('foo', 'bar');
+    });
+    expect(res).toEqual({ baz: 'baz' });
+  });
+});

lib/util/cache/package/sqlite.ts

@@ -0,0 +1,91 @@
+import Sqlite from 'better-sqlite3';
+import type { Database, Statement } from 'better-sqlite3';
+import * as upath from 'upath';
+import { logger } from '../../../logger';
+import { ensureDir } from '../../fs';
+
+export class SqlitePackageCache {
+  private readonly upsertStatement: Statement<unknown[]>;
+  private readonly getStatement: Statement<unknown[]>;
+  private readonly cleanupStatement: Statement<unknown[]>;
+
+  static async init(cacheDir: string): Promise<SqlitePackageCache> {
+    const sqliteDir = upath.join(cacheDir, '/renovate/renovate-cache-sqlite');
+    await ensureDir(sqliteDir);
+    const sqliteFile = upath.join(sqliteDir, 'db.sqlite');
+    const client = new Sqlite(sqliteFile);
+    const res = new SqlitePackageCache(client);
+    res.cleanup();
+    return res;
+  }
+
+  private constructor(private client: Database) {
+    client.pragma('journal_mode = WAL');
+    client.pragma("encoding = 'UTF-8'");
+
+    client
+      .prepare(
+        `
+          CREATE TABLE IF NOT EXISTS cache (
+            namespace TEXT NOT NULL,
+            key TEXT NOT NULL,
+            data TEXT NOT NULL,
+            expiry INTEGER NOT NULL,
+            PRIMARY KEY (namespace, key)
+          )
+        `,
+      )
+      .run();
+    client.prepare('CREATE INDEX IF NOT EXISTS expiry ON cache (expiry)').run();
+    client
+      .prepare(
+        'CREATE INDEX IF NOT EXISTS namespace_key ON cache (namespace, key)',
+      )
+      .run();
+
+    this.upsertStatement = client.prepare(`
+      INSERT INTO cache (namespace, key, data, expiry)
+      VALUES (@namespace, @key, @data, @expiry)
+      ON CONFLICT (namespace, key) DO UPDATE SET
+        data = @data,
+        expiry = @expiry
+    `);
+
+    this.getStatement = client
+      .prepare(
+        `
+          SELECT data FROM cache
+          WHERE
+            namespace = @namespace AND key = @key
+        `,
+      )
+      .pluck(true);
+
+    this.cleanupStatement = client.prepare(`
+      DELETE FROM cache
+      WHERE expiry <= @now
+    `);
+  }
+
+  set(namespace: string, key: string, value: unknown, ttlMinutes = 5): void {
+    const data = JSON.stringify(value);
+    const expiry = Date.now() + ttlMinutes * 60 * 1000;
+    this.upsertStatement.run({ namespace, key, data, expiry });
+  }
+
+  get<T = never>(namespace: string, key: string): T | undefined {
+    const res = this.getStatement.get({ namespace, key }) as string | undefined;
+    return res ? JSON.parse(res) : undefined;
+  }
+
+  private cleanup(): void {
+    const now = Date.now();
+    this.cleanupStatement.run({ now });
+    const timeMs = Date.now() - now;
+    logger.trace(`SQLite cache cleanup: ${timeMs}ms`);
+  }
+
+  close(): void {
+    this.client.close();
+  }
+}

lib/workers/repository/stats.spec.ts

@@ -173,12 +173,14 @@ describe('workers/repository/stats', () => {
             "count": 4,
             "maxMs": 100,
             "medianMs": 20,
+            "totalMs": 160,
           },
           "set": {
             "avgMs": 70,
             "count": 3,
             "maxMs": 110,
             "medianMs": 80,
+            "totalMs": 210,
           },
         }
       `);

lib/workers/repository/stats.ts

@@ -10,6 +10,7 @@ interface CacheStats {
   avgMs?: number;
   medianMs?: number;
   maxMs?: number;
+  totalMs?: number;
 }
 
 export function printLookupStats(): void {
@@ -46,8 +47,11 @@ export function printRequestStats(): void {
     },
   };
   if (packageCacheGets.length) {
+    packageCacheStats.get.totalMs = Math.round(
+      packageCacheGets.reduce((a, b) => a + b, 0),
+    );
     packageCacheStats.get.avgMs = Math.round(
-      packageCacheGets.reduce((a, b) => a + b, 0) / packageCacheGets.length,
+      packageCacheStats.get.totalMs / packageCacheGets.length,
     );
     if (packageCacheGets.length > 1) {
       packageCacheStats.get.medianMs =
@@ -57,8 +61,11 @@ export function printRequestStats(): void {
     }
   }
   if (packageCacheSets.length) {
+    packageCacheStats.set.totalMs = Math.round(
+      packageCacheSets.reduce((a, b) => a + b, 0),
+    );
     packageCacheStats.set.avgMs = Math.round(
-      packageCacheSets.reduce((a, b) => a + b, 0) / packageCacheSets.length,
+      packageCacheStats.set.totalMs / packageCacheSets.length,
     );
     if (packageCacheSets.length > 1) {
       packageCacheStats.set.medianMs =

package.json

@@ -168,6 +168,7 @@
     "@renovatebot/pep440": "3.0.17",
     "@renovatebot/ruby-semver": "3.0.22",
     "@sindresorhus/is": "4.6.0",
+    "@types/better-sqlite3": "7.6.8",
     "@types/ms": "0.7.34",
     "@types/tmp": "0.2.6",
     "@yarnpkg/core": "4.0.2",
@@ -177,6 +178,7 @@
     "auth-header": "1.0.0",
     "aws4": "1.12.0",
     "azure-devops-node-api": "12.1.0",
+    "better-sqlite3": "9.2.2",
     "bunyan": "1.8.15",
     "cacache": "18.0.2",
     "cacheable-lookup": "5.0.4",