Skip to content

Commit

Permalink
feat: cacheMode option (#267)
Browse files Browse the repository at this point in the history 
Co-authored-by: Samuel Attard <[email protected]>
Co-authored-by: Erick Zhao <[email protected]>
  • Loading branch information
@dsanders11 @MarshallOfSound @erickzhao
3 people authored Jul 15, 2024
1 parent 2898abb commit 0adc5b9
Show file tree
Hide file tree
Showing 6 changed files with 314 additions and 84 deletions.
172 changes: 108 additions & 64 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as sumchecker from 'sumchecker';
import { getArtifactFileName, getArtifactRemoteURL, getArtifactVersion } from './artifact-utils';
import {
ElectronArtifactDetails,
ElectronDownloadCacheMode,
ElectronDownloadRequestOptions,
ElectronGenericArtifactDetails,
ElectronPlatformArtifactDetails,
Expand All @@ -21,6 +22,11 @@ import {
getNodeArch,
ensureIsTruthyString,
isOfficialLinuxIA32Download,
mkdtemp,
doesCallerOwnTemporaryOutput,
effectiveCacheMode,
shouldTryReadCache,
TempDirCleanUpMode,
} from './utils';

export { getHostArch } from './utils';
Expand All @@ -42,58 +48,76 @@ async function validateArtifact(
downloadedAssetPath: string,
_downloadArtifact: ArtifactDownloader,
): Promise<void> {
return await withTempDirectoryIn(artifactDetails.tempDirectory, async tempFolder => {
// Don't try to verify the hash of the hash file itself
// and for older versions that don't have a SHASUMS256.txt
if (
!artifactDetails.artifactName.startsWith('SHASUMS256') &&
!artifactDetails.unsafelyDisableChecksums &&
semver.gte(artifactDetails.version, '1.3.2')
) {
let shasumPath: string;
const checksums = artifactDetails.checksums;
if (checksums) {
shasumPath = path.resolve(tempFolder, 'SHASUMS256.txt');
const fileNames: string[] = Object.keys(checksums);
if (fileNames.length === 0) {
throw new Error(
'Provided "checksums" object is empty, cannot generate a valid SHASUMS256.txt',
);
return await withTempDirectoryIn(
artifactDetails.tempDirectory,
async tempFolder => {
// Don't try to verify the hash of the hash file itself
// and for older versions that don't have a SHASUMS256.txt
if (
!artifactDetails.artifactName.startsWith('SHASUMS256') &&
!artifactDetails.unsafelyDisableChecksums &&
semver.gte(artifactDetails.version, '1.3.2')
) {
let shasumPath: string;
const checksums = artifactDetails.checksums;
if (checksums) {
shasumPath = path.resolve(tempFolder, 'SHASUMS256.txt');
const fileNames: string[] = Object.keys(checksums);
if (fileNames.length === 0) {
throw new Error(
'Provided "checksums" object is empty, cannot generate a valid SHASUMS256.txt',
);
}
const generatedChecksums = fileNames
.map(fileName => `${checksums[fileName]} *${fileName}`)
.join('\n');
await fs.writeFile(shasumPath, generatedChecksums);
} else {
shasumPath = await _downloadArtifact({
isGeneric: true,
version: artifactDetails.version,
artifactName: 'SHASUMS256.txt',
force: false,
downloadOptions: artifactDetails.downloadOptions,
cacheRoot: artifactDetails.cacheRoot,
downloader: artifactDetails.downloader,
mirrorOptions: artifactDetails.mirrorOptions,
// Never use the cache for loading checksums, load
// them fresh every time
cacheMode: ElectronDownloadCacheMode.Bypass,
});
}
const generatedChecksums = fileNames
.map(fileName => `${checksums[fileName]} *${fileName}`)
.join('\n');
await fs.writeFile(shasumPath, generatedChecksums);
} else {
shasumPath = await _downloadArtifact({
isGeneric: true,
version: artifactDetails.version,
artifactName: 'SHASUMS256.txt',
force: artifactDetails.force,
downloadOptions: artifactDetails.downloadOptions,
cacheRoot: artifactDetails.cacheRoot,
downloader: artifactDetails.downloader,
mirrorOptions: artifactDetails.mirrorOptions,
});
}

// For versions 1.3.2 - 1.3.4, need to overwrite the `defaultTextEncoding` option:
// https://github.com/electron/electron/pull/6676#discussion_r75332120
if (semver.satisfies(artifactDetails.version, '1.3.2 - 1.3.4')) {
const validatorOptions: sumchecker.ChecksumOptions = {};
validatorOptions.defaultTextEncoding = 'binary';
const checker = new sumchecker.ChecksumValidator('sha256', shasumPath, validatorOptions);
await checker.validate(
path.dirname(downloadedAssetPath),
path.basename(downloadedAssetPath),
);
} else {
await sumchecker('sha256', shasumPath, path.dirname(downloadedAssetPath), [
path.basename(downloadedAssetPath),
]);
try {
// For versions 1.3.2 - 1.3.4, need to overwrite the `defaultTextEncoding` option:
// https://github.com/electron/electron/pull/6676#discussion_r75332120
if (semver.satisfies(artifactDetails.version, '1.3.2 - 1.3.4')) {
const validatorOptions: sumchecker.ChecksumOptions = {};
validatorOptions.defaultTextEncoding = 'binary';
const checker = new sumchecker.ChecksumValidator(
'sha256',
shasumPath,
validatorOptions,
);
await checker.validate(
path.dirname(downloadedAssetPath),
path.basename(downloadedAssetPath),
);
} else {
await sumchecker('sha256', shasumPath, path.dirname(downloadedAssetPath), [
path.basename(downloadedAssetPath),
]);
}
} finally {
// Once we're done make sure we clean up the shasum temp dir
await fs.remove(path.dirname(shasumPath));
}
}
}
});
},
doesCallerOwnTemporaryOutput(effectiveCacheMode(artifactDetails))
? TempDirCleanUpMode.ORPHAN
: TempDirCleanUpMode.CLEAN,
);
}

/**
Expand Down Expand Up @@ -133,21 +157,33 @@ export async function downloadArtifact(
const fileName = getArtifactFileName(details);
const url = await getArtifactRemoteURL(details);
const cache = new Cache(details.cacheRoot);
const cacheMode = effectiveCacheMode(details);

// Do not check if the file exists in the cache when force === true
if (!details.force) {
if (shouldTryReadCache(cacheMode)) {
d(`Checking the cache (${details.cacheRoot}) for ${fileName} (${url})`);
const cachedPath = await cache.getPathForFileInCache(url, fileName);

if (cachedPath === null) {
d('Cache miss');
} else {
d('Cache hit');
let artifactPath = cachedPath;
if (doesCallerOwnTemporaryOutput(cacheMode)) {
// Copy out of cache into temporary directory if readOnly cache so
// that the caller can take ownership of the returned file
const tempDir = await mkdtemp(artifactDetails.tempDirectory);
artifactPath = path.resolve(tempDir, fileName);
await fs.copyFile(cachedPath, artifactPath);
}
try {
await validateArtifact(details, cachedPath, downloadArtifact);
await validateArtifact(details, artifactPath, downloadArtifact);

return cachedPath;
return artifactPath;
} catch (err) {
if (doesCallerOwnTemporaryOutput(cacheMode)) {
await fs.remove(path.dirname(artifactPath));
}
d("Artifact in cache didn't match checksums", err);
d('falling back to re-download');
}
Expand All @@ -167,21 +203,29 @@ export async function downloadArtifact(
console.warn('For more info: https://electronjs.org/blog/linux-32bit-support');
}

return await withTempDirectoryIn(details.tempDirectory, async tempFolder => {
const tempDownloadPath = path.resolve(tempFolder, getArtifactFileName(details));
return await withTempDirectoryIn(
details.tempDirectory,
async tempFolder => {
const tempDownloadPath = path.resolve(tempFolder, getArtifactFileName(details));

const downloader = details.downloader || (await getDownloaderForSystem());
d(
`Downloading ${url} to ${tempDownloadPath} with options: ${JSON.stringify(
details.downloadOptions,
)}`,
);
await downloader.download(url, tempDownloadPath, details.downloadOptions);
const downloader = details.downloader || (await getDownloaderForSystem());
d(
`Downloading ${url} to ${tempDownloadPath} with options: ${JSON.stringify(
details.downloadOptions,
)}`,
);
await downloader.download(url, tempDownloadPath, details.downloadOptions);

await validateArtifact(details, tempDownloadPath, downloadArtifact);
await validateArtifact(details, tempDownloadPath, downloadArtifact);

return await cache.putFileInCache(url, tempDownloadPath, fileName);
});
if (doesCallerOwnTemporaryOutput(cacheMode)) {
return tempDownloadPath;
} else {
return await cache.putFileInCache(url, tempDownloadPath, fileName);
}
},
doesCallerOwnTemporaryOutput(cacheMode) ? TempDirCleanUpMode.ORPHAN : TempDirCleanUpMode.CLEAN,
);
}

/**
Expand Down
41 changes: 41 additions & 0 deletions src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,28 @@ export interface ElectronDownloadRequest {
artifactName: string;
}

export enum ElectronDownloadCacheMode {
/**
* Reads from the cache if present
* Writes to the cache after fetch if not present
*/
ReadWrite,
/**
* Reads from the cache if present
* Will **not** write back to the cache after fetching missing artifact
*/
ReadOnly,
/**
* Skips reading from the cache
* Will write back into the cache, overwriting anything currently in the cache after fetch
*/
WriteOnly,
/**
* Bypasses the cache completely, neither reads from nor writes to the cache
*/
Bypass,
}

/**
* @category Download Electron
*/
Expand All @@ -90,6 +112,7 @@ export interface ElectronDownloadRequestOptions {
* Whether to download an artifact regardless of whether it's in the cache directory.
*
* @defaultValue `false`
* @deprecated This option is deprecated and directly maps to {@link cacheMode | `cacheMode: ElectronDownloadCacheMode.WriteOnly`}
*/
force?: boolean;
/**
Expand Down Expand Up @@ -148,6 +171,24 @@ export interface ElectronDownloadRequestOptions {
* @defaultValue the OS default temporary directory via [`os.tmpdir()`](https://nodejs.org/api/os.html#ostmpdir)
*/
tempDirectory?: string;
/**
* Controls the cache read and write behavior.
*
* When set to either {@link ElectronDownloadCacheMode.ReadOnly | ReadOnly} or
* {@link ElectronDownloadCacheMode.Bypass | Bypass}, the caller is responsible
* for cleaning up the returned file path once they are done using it
* (e.g. via `fs.remove(path.dirname(pathFromElectronGet))`).
*
* When set to either {@link ElectronDownloadCacheMode.WriteOnly | WriteOnly} or
* {@link ElectronDownloadCacheMode.ReadWrite | ReadWrite} (the default), the caller
* should not move or delete the file path that is returned as the path
* points directly to the disk cache.
*
* This option cannot be used in conjunction with {@link ElectronDownloadRequestOptions.force}.
*
* @defaultValue {@link ElectronDownloadCacheMode.ReadWrite}
*/
cacheMode?: ElectronDownloadCacheMode;
}

/**
Expand Down
68 changes: 63 additions & 5 deletions src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ import * as childProcess from 'child_process';
import * as fs from 'fs-extra';
import * as os from 'os';
import * as path from 'path';
import {
ElectronDownloadCacheMode,
ElectronGenericArtifactDetails,
ElectronPlatformArtifactDetailsWithDefaults,
} from './types';

async function useAndRemoveDirectory<T>(
directory: string,
Expand All @@ -16,17 +21,34 @@ async function useAndRemoveDirectory<T>(
return result;
}

export async function mkdtemp(parentDirectory: string = os.tmpdir()): Promise<string> {
const tempDirectoryPrefix = 'electron-download-';
return await fs.mkdtemp(path.resolve(parentDirectory, tempDirectoryPrefix));
}

export enum TempDirCleanUpMode {
CLEAN,
ORPHAN,
}

export async function withTempDirectoryIn<T>(
parentDirectory: string = os.tmpdir(),
fn: (directory: string) => Promise<T>,
cleanUp: TempDirCleanUpMode,
): Promise<T> {
const tempDirectoryPrefix = 'electron-download-';
const tempDirectory = await fs.mkdtemp(path.resolve(parentDirectory, tempDirectoryPrefix));
return useAndRemoveDirectory(tempDirectory, fn);
const tempDirectory = await mkdtemp(parentDirectory);
if (cleanUp === TempDirCleanUpMode.CLEAN) {
return useAndRemoveDirectory(tempDirectory, fn);
} else {
return fn(tempDirectory);
}
}

export async function withTempDirectory<T>(fn: (directory: string) => Promise<T>): Promise<T> {
return withTempDirectoryIn(undefined, fn);
export async function withTempDirectory<T>(
fn: (directory: string) => Promise<T>,
cleanUp: TempDirCleanUpMode,
): Promise<T> {
return withTempDirectoryIn(undefined, fn, cleanUp);
}

export function normalizeVersion(version: string): string {
Expand Down Expand Up @@ -122,3 +144,39 @@ export function setEnv(key: string, value: string | undefined): void {
process.env[key] = value;
}
}

export function effectiveCacheMode(
artifactDetails: ElectronPlatformArtifactDetailsWithDefaults | ElectronGenericArtifactDetails,
): ElectronDownloadCacheMode {
if (artifactDetails.force) {
if (artifactDetails.cacheMode) {
throw new Error(
'Setting both "force" and "cacheMode" is not supported, please exclusively use "cacheMode"',
);
}
return ElectronDownloadCacheMode.WriteOnly;
}

return artifactDetails.cacheMode || ElectronDownloadCacheMode.ReadWrite;
}

export function shouldTryReadCache(cacheMode: ElectronDownloadCacheMode): boolean {
return (
cacheMode === ElectronDownloadCacheMode.ReadOnly ||
cacheMode === ElectronDownloadCacheMode.ReadWrite
);
}

export function shouldWriteCache(cacheMode: ElectronDownloadCacheMode): boolean {
return (
cacheMode === ElectronDownloadCacheMode.WriteOnly ||
cacheMode === ElectronDownloadCacheMode.ReadWrite
);
}

export function doesCallerOwnTemporaryOutput(cacheMode: ElectronDownloadCacheMode): boolean {
return (
cacheMode === ElectronDownloadCacheMode.Bypass ||
cacheMode === ElectronDownloadCacheMode.ReadOnly
);
}
Loading

0 comments on commit 0adc5b9

Please sign in to comment.