support for multiple TUF repo caches

Signed-off-by: Brian DeHamer <[email protected]>

.changeset/sharp-rabbits-fix.md

@@ -0,0 +1,5 @@
+---
+"@sigstore/tuf": minor
+---
+
+Add support for caching metadata from multiple TUF repositories

packages/tuf/package.json

@@ -10,8 +10,7 @@
     "test": "jest"
   },
   "files": [
-    "dist",
-    "store"
+    "dist"
   ],
   "author": "[email protected]",
   "license": "Apache-2.0",

packages/tuf/src/__tests__/client.test.ts

@@ -20,35 +20,51 @@ import os from 'os';
 import path from 'path';
 import { TUFClient, TUFOptions } from '../client';
 import { TUFError } from '../error';
+import { REPO_SEEDS } from '../store';
 
 describe('TUFClient', () => {
-  const rootPath = require.resolve(
-    '../../store/public-good-instance-root.json'
-  );
-
   describe('constructor', () => {
-    const cacheDir = path.join(os.tmpdir(), 'tuf-client-test');
-    const mirrorURL = 'https://example.com';
+    let rootSeedDir: string;
+    let rootPath: string;
+
+    const repoName = 'example.com';
+    const mirrorURL = `https://${repoName}`;
+    const cacheRoot = path.join(os.tmpdir(), 'tuf-client-test');
+    const cacheDir = path.join(cacheRoot, repoName);
     const force = false;
-    afterEach(() => fs.rmSync(cacheDir, { recursive: true }));
+
+    beforeEach(() => {
+      rootSeedDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), 'tuf-client-test-seed')
+      );
+
+      rootPath = path.join(rootSeedDir, 'root-seed.json');
+      fs.writeFileSync(
+        rootPath,
+        Buffer.from(
+          REPO_SEEDS['https://tuf-repo-cdn.sigstore.dev']['root.json'],
+          'base64'
+        )
+      );
+    });
+
+    afterEach(() => {
+      fs.rmSync(cacheRoot, { recursive: true });
+      fs.rmSync(rootSeedDir, { recursive: true });
+    });
 
     describe('when the cache directory does not exist', () => {
       it('creates the cache directory', () => {
-        new TUFClient({ cachePath: cacheDir, mirrorURL, rootPath, force });
+        new TUFClient({ cachePath: cacheRoot, mirrorURL, rootPath, force });
         expect(fs.existsSync(cacheDir)).toEqual(true);
         expect(fs.existsSync(path.join(cacheDir, 'root.json'))).toEqual(true);
-        expect(fs.existsSync(path.join(cacheDir, 'remote.json'))).toEqual(true);
       });
     });
 
     describe('when the cache directory already exists', () => {
       beforeEach(() => {
         fs.mkdirSync(cacheDir, { recursive: true });
         fs.copyFileSync(rootPath, path.join(cacheDir, 'root.json'));
-        fs.writeFileSync(
-          path.join(cacheDir, 'remote.json'),
-          JSON.stringify({ mirror: mirrorURL })
-        );
       });
 
       it('loads config from the existing directory without error', () => {
@@ -59,45 +75,47 @@ describe('TUFClient', () => {
       });
     });
 
+    describe('when no explicit root.json is provided', () => {
+      describe('when the mirror URL does NOT match one of the embedded roots', () => {
+        const mirrorURL = 'https://oops.net';
+        it('throws an error', () => {
+          expect(
+            () => new TUFClient({ cachePath: cacheDir, mirrorURL, force })
+          ).toThrowWithCode(TUFError, 'TUF_INIT_CACHE_ERROR');
+        });
+      });
+
+      describe('when the mirror URL matches one of the embedded roots', () => {
+        const mirrorURL = 'https://tuf-repo-cdn.sigstore.dev';
+        it('loads the embedded root.json', () => {
+          expect(
+            () => new TUFClient({ cachePath: cacheDir, mirrorURL, force })
+          ).not.toThrow();
+        });
+      });
+    });
+
     describe('when forcing re-initialization of an existing directory', () => {
-      const oldMirrorURL = mirrorURL;
-      const newMirrorURL = 'https://new.org';
       const force = true;
 
       beforeEach(() => {
         fs.mkdirSync(cacheDir, { recursive: true });
-        fs.copyFileSync(rootPath, path.join(cacheDir, 'root.json'));
-        fs.writeFileSync(
-          path.join(cacheDir, 'remote.json'),
-          JSON.stringify({ mirror: oldMirrorURL })
-        );
+        fs.writeFileSync(path.join(cacheDir, 'root.json'), 'oops');
       });
 
       it('initializes the client without error', () => {
         expect(
           () =>
-            new TUFClient({
-              cachePath: cacheDir,
-              mirrorURL: newMirrorURL,
-              rootPath,
-              force,
-            })
+            new TUFClient({ cachePath: cacheRoot, mirrorURL, rootPath, force })
         ).not.toThrow();
       });
 
       it('overwrites the existing values', () => {
-        new TUFClient({
-          cachePath: cacheDir,
-          mirrorURL: newMirrorURL,
-          rootPath,
-          force,
-        });
+        new TUFClient({ cachePath: cacheRoot, mirrorURL, rootPath, force });
 
-        const remote = fs.readFileSync(
-          path.join(cacheDir, 'remote.json'),
-          'utf-8'
-        );
-        expect(JSON.parse(remote)).toEqual({ mirror: newMirrorURL });
+        const root = fs.readFileSync(path.join(cacheDir, 'root.json'), 'utf-8');
+        expect(root).toBeDefined();
+        expect(root).not.toEqual('oops');
       });
     });
   });
@@ -117,7 +135,7 @@ describe('TUFClient', () => {
         mirrorURL: tufRepo.baseURL,
         cachePath: tufRepo.cachePath,
         retry: false,
-        rootPath,
+        rootPath: path.join(tufRepo.cachePath, 'root.json'),
         force: false,
       };
     });

packages/tuf/src/__tests__/index.test.ts

@@ -17,6 +17,7 @@ limitations under the License.
 import { TrustedRoot } from '@sigstore/protobuf-specs';
 import mocktuf, { Target } from '@tufjs/repo-mock';
 import fs from 'fs';
+import path from 'path';
 import { TUF, TUFError, TUFOptions, getTrustedRoot, initTUF } from '..';
 import { TUFClient } from '../client';
 
@@ -73,7 +74,7 @@ describe('getTrustedRoot', () => {
       mirrorURL: tufRepo.baseURL,
       cachePath: tufRepo.cachePath,
       retry: false,
-      rootPath: 'n/a',
+      rootPath: path.join(tufRepo.cachePath, 'root.json'),
     };
   });
 
@@ -96,27 +97,26 @@ describe('initTUF', () => {
       mirrorURL: tufRepo.baseURL,
       cachePath: tufRepo.cachePath,
       retry: false,
-      rootPath: 'n/a',
+      rootPath: path.join(tufRepo.cachePath, 'root.json'),
     };
   });
 
   afterEach(() => tufRepo?.teardown());
 
-  it('returns a TUFCLient', async () => {
+  it('returns a TUFClient', async () => {
     const tuf = await initTUF(options);
     expect(tuf).toBeInstanceOf(TUFClient);
   });
 
   it('sets-up the local TUF cache', async () => {
     await initTUF(options);
 
-    const cachePath = tufRepo!.cachePath;
+    const cachePath = path.join(
+      tufRepo!.cachePath,
+      new URL(options!.mirrorURL!).host
+    );
     expect(fs.existsSync(cachePath)).toBe(true);
-    expect(fs.existsSync(`${cachePath}/remote.json`)).toBe(true);
     expect(fs.existsSync(`${cachePath}/root.json`)).toBe(true);
-    expect(fs.existsSync(`${cachePath}/snapshot.json`)).toBe(true);
-    expect(fs.existsSync(`${cachePath}/timestamp.json`)).toBe(true);
-    expect(fs.existsSync(`${cachePath}/targets.json`)).toBe(true);
     expect(fs.existsSync(`${cachePath}/targets`)).toBe(true);
   });
 });

packages/tuf/src/client.ts

@@ -16,12 +16,16 @@ limitations under the License.
 import fs from 'fs';
 import path from 'path';
 import { Config, Updater } from 'tuf-js';
+import { TUFError } from '.';
+import { REPO_SEEDS } from './store';
 import { readTarget } from './target';
 
 import type { MakeFetchHappenOptions } from 'make-fetch-happen';
 
 export type Retry = MakeFetchHappenOptions['retry'];
 
+const TARGETS_DIR_NAME = 'targets';
+
 type FetchOptions = {
   retry?: Retry;
   timeout?: number;
@@ -30,25 +34,33 @@ type FetchOptions = {
 export type TUFOptions = {
   cachePath: string;
   mirrorURL: string;
-  rootPath: string;
+  rootPath?: string;
   force: boolean;
 } & FetchOptions;
 
 export interface TUF {
   getTarget(targetName: string): Promise<string>;
 }
 
-interface RemoteConfig {
-  mirror: string;
-}
-
 export class TUFClient implements TUF {
   private updater: Updater;
 
   constructor(options: TUFOptions) {
-    initTufCache(options);
-    const remote = initRemoteConfig(options);
-    this.updater = initClient(options.cachePath, remote, options);
+    const url = new URL(options.mirrorURL);
+    const repoName = encodeURIComponent(
+      url.host + url.pathname.replace(/\/$/, '')
+    );
+    const cachePath = path.join(options.cachePath, repoName);
+
+    initTufCache(cachePath);
+    seedCache({
+      cachePath,
+      mirrorURL: options.mirrorURL,
+      tufRootPath: options.rootPath,
+      force: options.force,
+    });
+
+    this.updater = initClient(options.mirrorURL, cachePath, options);
   }
 
   public async refresh(): Promise<void> {
@@ -65,13 +77,8 @@ export class TUFClient implements TUF {
 // created. If the targets directory does not exist, it will be created.
 // If the root.json file does not exist, it will be copied from the
 // rootPath argument.
-function initTufCache({
-  cachePath,
-  rootPath: tufRootPath,
-  force,
-}: TUFOptions): string {
-  const targetsPath = path.join(cachePath, 'targets');
-  const cachedRootPath = path.join(cachePath, 'root.json');
+function initTufCache(cachePath: string): void {
+  const targetsPath = path.join(cachePath, TARGETS_DIR_NAME);
 
   if (!fs.existsSync(cachePath)) {
     fs.mkdirSync(cachePath, { recursive: true });
@@ -80,60 +87,70 @@ function initTufCache({
   if (!fs.existsSync(targetsPath)) {
     fs.mkdirSync(targetsPath);
   }
-
-  // If the root.json file does not exist (or we're forcing re-initialization),
-  // copy it from the rootPath argument
-  if (!fs.existsSync(cachedRootPath) || force) {
-    fs.copyFileSync(tufRootPath, cachedRootPath);
-  }
-
-  return cachePath;
 }
 
-// Initializes the remote.json file, which contains the URL of the TUF
-// repository. If the file does not exist, it will be created. If the file
-// exists, it will be parsed and returned.
-function initRemoteConfig({
+// Populates the TUF cache with the initial root.json file. If the root.json
+// file does not exist (or we're forcing re-initialization), copy it from either
+// the rootPath argument or from one of the repo seeds.
+function seedCache({
   cachePath,
   mirrorURL,
+  tufRootPath,
   force,
-}: TUFOptions): RemoteConfig {
-  let remoteConfig: RemoteConfig | undefined;
-  const remoteConfigPath = path.join(cachePath, 'remote.json');
-
-  // If the remote config file exists, read it and parse it (skip if force is
-  // true)
-  if (!force && fs.existsSync(remoteConfigPath)) {
-    const data = fs.readFileSync(remoteConfigPath, 'utf-8');
-    remoteConfig = JSON.parse(data);
-  }
+}: {
+  cachePath: string;
+  mirrorURL: string;
+  tufRootPath?: string;
+  force: boolean;
+}): void {
+  const cachedRootPath = path.join(cachePath, 'root.json');
 
-  // If the remote config file does not exist (or we're forcing initialization),
-  // create it
-  if (!remoteConfig || force) {
-    remoteConfig = { mirror: mirrorURL };
-    fs.writeFileSync(remoteConfigPath, JSON.stringify(remoteConfig));
+  // If the root.json file does not exist (or we're forcing re-initialization),
+  // populate it either from the supplied rootPath or from one of the repo seeds.
+  if (!fs.existsSync(cachedRootPath) || force) {
+    if (tufRootPath) {
+      fs.copyFileSync(tufRootPath, cachedRootPath);
+    } else {
+      const repoSeed = REPO_SEEDS[mirrorURL];
+
+      if (!repoSeed) {
+        throw new TUFError({
+          code: 'TUF_INIT_CACHE_ERROR',
+          message: `No root.json found for mirror: ${mirrorURL}`,
+        });
+      }
+
+      fs.writeFileSync(
+        cachedRootPath,
+        Buffer.from(repoSeed['root.json'], 'base64')
+      );
+
+      // Copy any seed targets into the cache
+      Object.entries(repoSeed.targets).forEach(([targetName, target]) => {
+        fs.writeFileSync(
+          path.join(cachePath, TARGETS_DIR_NAME, targetName),
+          Buffer.from(target, 'base64')
+        );
+      });
+    }
   }
-
-  return remoteConfig;
 }
 
 function initClient(
+  mirrorURL: string,
   cachePath: string,
-  remote: RemoteConfig,
   options: FetchOptions
 ): Updater {
-  const baseURL = remote.mirror;
   const config: Partial<Config> = {
     fetchTimeout: options.timeout,
     fetchRetry: options.retry,
   };
 
   return new Updater({
-    metadataBaseUrl: baseURL,
-    targetBaseUrl: `${baseURL}/targets`,
+    metadataBaseUrl: mirrorURL,
+    targetBaseUrl: `${mirrorURL}/targets`,
     metadataDir: cachePath,
-    targetDir: path.join(cachePath, 'targets'),
+    targetDir: path.join(cachePath, TARGETS_DIR_NAME),
     config,
   });
 }