diff --git a/packages/sanity/package.json b/packages/sanity/package.json index 2222c01f0f1..ace10d484be 100644 --- a/packages/sanity/package.json +++ b/packages/sanity/package.json @@ -231,7 +231,9 @@ "@types/tar-stream": "^3.1.3", "@types/use-sync-external-store": "^0.0.5", "@vitejs/plugin-react": "^4.2.0", + "archiver": "^6.0.1", "arrify": "^1.0.1", + "async-mutex": "^0.4.1", "chalk": "^4.1.2", "chokidar": "^3.5.3", "classnames": "^2.2.5", @@ -310,6 +312,7 @@ "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", "@testing-library/user-event": "^13.0.16", + "@types/archiver": "^6.0.2", "@types/arrify": "^1.0.4", "@types/connect-history-api-fallback": "^1.5.2", "@types/lodash": "^4.14.149", diff --git a/packages/sanity/src/_internal/cli/actions/backup/archiveDir.ts b/packages/sanity/src/_internal/cli/actions/backup/archiveDir.ts new file mode 100644 index 00000000000..89aeb1f4cb2 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/archiveDir.ts @@ -0,0 +1,51 @@ +import {createWriteStream} from 'node:fs' +import zlib from 'node:zlib' + +import {type ProgressData} from 'archiver' + +import debug from './debug' + +const archiver = require('archiver') + +// ProgressCb is a callback that is called with the number of bytes processed so far. +type ProgressCb = (processedBytes: number) => void + +// archiveDir creates a tarball of the given directory and writes it to the given file path. +function archiveDir(tmpOutDir: string, outFilePath: string, progressCb: ProgressCb): Promise { + return new Promise((resolve, reject) => { + const archiveDestination = createWriteStream(outFilePath) + archiveDestination.on('error', (err: Error) => { + reject(err) + }) + + archiveDestination.on('close', () => { + resolve() + }) + + const archive = archiver('tar', { + gzip: true, + gzipOptions: {level: zlib.constants.Z_DEFAULT_COMPRESSION}, + }) + + archive.on('error', (err: Error) => { + debug('Archiving errored!\n%s', err.stack) + reject(err) + }) + + // Catch warnings for non-blocking errors (stat failures and others) + archive.on('warning', (err: Error) => { + debug('Archive warning: %s', err.message) + }) + + archive.on('progress', (progress: ProgressData) => { + progressCb(progress.fs.processedBytes) + }) + + // Pipe archive data to the file + archive.pipe(archiveDestination) + archive.directory(tmpOutDir, false) + archive.finalize() + }) +} + +export default archiveDir diff --git a/packages/sanity/src/_internal/cli/actions/backup/chooseBackupIdPrompt.ts b/packages/sanity/src/_internal/cli/actions/backup/chooseBackupIdPrompt.ts new file mode 100644 index 00000000000..55fefdb8187 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/chooseBackupIdPrompt.ts @@ -0,0 +1,46 @@ +import {type CliCommandContext} from '@sanity/cli' + +import {defaultApiVersion} from '../../commands/backup/backupGroup' +import resolveApiClient from './resolveApiClient' + +// maxBackupIdsShown is the maximum number of backup IDs to show in the prompt. +// Higher numbers will cause the prompt to be slow. +const maxBackupIdsShown = 100 + +async function chooseBackupIdPrompt( + context: CliCommandContext, + datasetName: string, +): Promise { + const {prompt} = context + + const {projectId, token, client} = await resolveApiClient(context, datasetName, defaultApiVersion) + + try { + // Fetch last $maxBackupIdsShown backups for this dataset. + // We expect here that API returns backups sorted by creation date in descending order. + const response = await client.request({ + headers: {Authorization: `Bearer ${token}`}, + uri: `/projects/${projectId}/datasets/${datasetName}/backups`, + query: {limit: maxBackupIdsShown.toString()}, + }) + + if (response?.backups?.length > 0) { + const backupIdChoices = response.backups.map((backup: {id: string}) => ({ + value: backup.id, + })) + const selected = await prompt.single({ + message: `Select backup ID to use (only last ${maxBackupIdsShown} shown)`, + type: 'list', + choices: backupIdChoices, + }) + + return selected + } + } catch (err) { + throw new Error(`Failed to fetch backups for dataset ${datasetName}: ${err.message}`) + } + + throw new Error('No backups found') +} + +export default chooseBackupIdPrompt diff --git a/packages/sanity/src/_internal/cli/actions/backup/cleanupTmpDir.ts b/packages/sanity/src/_internal/cli/actions/backup/cleanupTmpDir.ts new file mode 100644 index 00000000000..f32dc4b203d --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/cleanupTmpDir.ts @@ -0,0 +1,13 @@ +import rimraf from 'rimraf' + +import debug from './debug' + +function cleanupTmpDir(tmpDir: string): void { + rimraf(tmpDir, (err) => { + if (err) { + debug(`Error cleaning up temporary files: ${err.message}`) + } + }) +} + +export default cleanupTmpDir diff --git a/packages/sanity/src/_internal/cli/actions/backup/debug.ts b/packages/sanity/src/_internal/cli/actions/backup/debug.ts new file mode 100644 index 00000000000..cec0c1595fa --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/debug.ts @@ -0,0 +1 @@ +export default require('debug')('sanity:backup') diff --git a/packages/sanity/src/_internal/cli/actions/backup/downloadAsset.ts b/packages/sanity/src/_internal/cli/actions/backup/downloadAsset.ts new file mode 100644 index 00000000000..99268e8b0f3 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/downloadAsset.ts @@ -0,0 +1,54 @@ +import {createWriteStream} from 'node:fs' +import path from 'node:path' + +import {getIt} from 'get-it' +import {keepAlive, promise} from 'get-it/middleware' + +import debug from './debug' +import withRetry from './withRetry' + +const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds +const READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes + +const request = getIt([keepAlive(), promise()]) + +async function downloadAsset( + url: string, + fileName: string, + fileType: string, + outDir: string, +): Promise { + // File names that contain a path to file (e.g. sanity-storage/assets/file-name.tar.gz) fail when archive is + // created due to missing parent dir (e.g. sanity-storage/assets), so we want to handle them by taking + // the base name as file name. + const normalizedFileName = path.basename(fileName) + + const assetFilePath = getAssetFilePath(normalizedFileName, fileType, outDir) + await withRetry(async () => { + const response = await request({ + url: url, + maxRedirects: 5, + timeout: {connect: CONNECTION_TIMEOUT, socket: READ_TIMEOUT}, + stream: true, + }) + + debug('Received asset %s with status code %d', normalizedFileName, response?.statusCode) + + response.body.pipe(createWriteStream(assetFilePath)) + }) +} + +function getAssetFilePath(fileName: string, fileType: string, outDir: string): string { + // Set assetFilePath if we are downloading an asset file. + // If it's a JSON document, assetFilePath will be an empty string. + let assetFilePath = '' + if (fileType === 'image') { + assetFilePath = path.join(outDir, 'images', fileName) + } else if (fileType === 'file') { + assetFilePath = path.join(outDir, 'files', fileName) + } + + return assetFilePath +} + +export default downloadAsset diff --git a/packages/sanity/src/_internal/cli/actions/backup/downloadDocument.ts b/packages/sanity/src/_internal/cli/actions/backup/downloadDocument.ts new file mode 100644 index 00000000000..3267b92c659 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/downloadDocument.ts @@ -0,0 +1,27 @@ +import {getIt, type MiddlewareResponse} from 'get-it' +import {keepAlive, promise} from 'get-it/middleware' + +import debug from './debug' +import withRetry from './withRetry' + +const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds +const READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes + +const request = getIt([keepAlive(), promise()]) + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function downloadDocument(url: string): Promise { + const response = await withRetry(() => + request({ + url, + maxRedirects: 5, + timeout: {connect: CONNECTION_TIMEOUT, socket: READ_TIMEOUT}, + }), + ) + + debug('Received document from %s with status code %d', url, response?.statusCode) + + return response.body +} + +export default downloadDocument diff --git a/packages/sanity/src/_internal/cli/actions/backup/fetchNextBackupPage.ts b/packages/sanity/src/_internal/cli/actions/backup/fetchNextBackupPage.ts new file mode 100644 index 00000000000..d9c4d288591 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/fetchNextBackupPage.ts @@ -0,0 +1,88 @@ +import {Readable} from 'node:stream' + +import {type QueryParams, type SanityClient} from '@sanity/client' + +type File = { + name: string + url: string + type: string +} + +type GetBackupResponse = { + createdAt: string + totalFiles: number + files: File[] + nextCursor?: string +} + +class PaginatedGetBackupStream extends Readable { + private cursor = '' + private readonly client: SanityClient + private readonly projectId: string + private readonly datasetName: string + private readonly backupId: string + private readonly token: string + public totalFiles = 0 + + constructor( + client: SanityClient, + projectId: string, + datasetName: string, + backupId: string, + token: string, + ) { + super({objectMode: true}) + this.client = client + this.projectId = projectId + this.datasetName = datasetName + this.backupId = backupId + this.token = token + } + + async _read(): Promise { + try { + const data = await this.fetchNextBackupPage() + + // Set totalFiles when it's fetched for the first time + if (this.totalFiles === 0) { + this.totalFiles = data.totalFiles + } + + data.files.forEach((file: File) => this.push(file)) + + if (typeof data.nextCursor === 'string' && data.nextCursor !== '') { + this.cursor = data.nextCursor + } else { + // No more pages left to fetch. + this.push(null) + } + } catch (err) { + this.destroy(err as Error) + } + } + + // fetchNextBackupPage fetches the next page of backed up files from the backup API. + async fetchNextBackupPage(): Promise { + const query: QueryParams = this.cursor === '' ? {} : {nextCursor: this.cursor} + + try { + return await this.client.request({ + headers: {Authorization: `Bearer ${this.token}`}, + uri: `/projects/${this.projectId}/datasets/${this.datasetName}/backups/${this.backupId}`, + query, + }) + } catch (error) { + // It can be clearer to pull this logic out in a common error handling function for re-usability. + let msg = error.statusCode ? error.response.body.message : error.message + + // If no message can be extracted, print the whole error. + if (msg === undefined) { + msg = String(error) + } + throw new Error(`Downloading dataset backup failed: ${msg}`) + } + } +} + +export {PaginatedGetBackupStream} +export type {File, GetBackupResponse} diff --git a/packages/sanity/src/_internal/cli/actions/backup/parseApiErr.ts b/packages/sanity/src/_internal/cli/actions/backup/parseApiErr.ts new file mode 100644 index 00000000000..92ef01fcb3c --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/parseApiErr.ts @@ -0,0 +1,35 @@ +// apiErr is a type that represents an error returned by the API +interface ApiErr { + statusCode: number + message: string +} + +// parseApiErr is a function that attempts with the best effort to parse +// an error returned by the API since different API endpoint may end up +// returning different error structures. +// eslint-disable-next-line @typescript-eslint/no-explicit-any,@typescript-eslint/explicit-module-boundary-types +function parseApiErr(err: any): ApiErr { + const apiErr = {} as ApiErr + if (err.code) { + apiErr.statusCode = err.code + } else if (err.statusCode) { + apiErr.statusCode = err.statusCode + } + + if (err.message) { + apiErr.message = err.message + } else if (err.statusMessage) { + apiErr.message = err.statusMessage + } else if (err?.response?.body?.message) { + apiErr.message = err.response.body.message + } else if (err?.response?.data?.message) { + apiErr.message = err.response.data.message + } else { + // If no message can be extracted, print the whole error. + apiErr.message = JSON.stringify(err) + } + + return apiErr +} + +export default parseApiErr diff --git a/packages/sanity/src/_internal/cli/actions/backup/progressSpinner.ts b/packages/sanity/src/_internal/cli/actions/backup/progressSpinner.ts new file mode 100644 index 00000000000..9009d32a00a --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/progressSpinner.ts @@ -0,0 +1,59 @@ +import {type CliOutputter} from '@sanity/cli' +import prettyMs from 'pretty-ms' + +type ProgressEvent = { + step: string + update?: boolean + current?: number + total?: number +} + +interface ProgressSpinner { + set: (progress: ProgressEvent) => void + update: (progress: ProgressEvent) => void + succeed: () => void + fail: () => void +} + +const newProgress = (output: CliOutputter, startStep: string): ProgressSpinner => { + let spinner = output.spinner(startStep).start() + let lastProgress: ProgressEvent = {step: startStep} + let start = Date.now() + + const print = (progress: ProgressEvent) => { + const elapsed = prettyMs(Date.now() - start) + if (progress.current && progress.current > 0 && progress.total && progress.total > 0) { + spinner.text = `${progress.step} (${progress.current}/${progress.total}) [${elapsed}]` + } else { + spinner.text = `${progress.step} [${elapsed}]` + } + } + + return { + set: (progress: ProgressEvent) => { + if (progress.step !== lastProgress.step) { + print(lastProgress) // Print the last progress before moving on + spinner.succeed() + spinner = output.spinner(progress.step).start() + start = Date.now() + } else if (progress.step === lastProgress.step && progress.update) { + print(progress) + } + lastProgress = progress + }, + update: (progress: ProgressEvent) => { + print(progress) + lastProgress = progress + }, + succeed: () => { + spinner.succeed() + start = Date.now() + }, + fail: () => { + spinner.fail() + start = Date.now() + }, + } +} + +export default newProgress diff --git a/packages/sanity/src/_internal/cli/actions/backup/resolveApiClient.ts b/packages/sanity/src/_internal/cli/actions/backup/resolveApiClient.ts new file mode 100644 index 00000000000..bda841094f5 --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/resolveApiClient.ts @@ -0,0 +1,46 @@ +import {type CliCommandContext} from '@sanity/cli' +import {type SanityClient} from '@sanity/client' + +import {chooseDatasetPrompt} from '../dataset/chooseDatasetPrompt' + +type ResolvedApiClient = { + projectId: string + datasetName: string + token?: string + client: SanityClient +} + +async function resolveApiClient( + context: CliCommandContext, + datasetName: string, + apiVersion: string, +): Promise { + const {apiClient} = context + + let client = apiClient() + const {projectId, token} = client.config() + + if (!projectId) { + throw new Error('Project ID not defined') + } + + // If no dataset provided, explicitly ask for dataset instead of using dataset + // configured in Sanity config. Aligns with `sanity dataset export` behavior. + let selectedDataset: string = datasetName + if (!selectedDataset) { + selectedDataset = await chooseDatasetPrompt(context, { + message: 'Select the dataset name:', + }) + } + + client = client.withConfig({dataset: datasetName, apiVersion}) + + return { + projectId, + datasetName: selectedDataset, + token, + client, + } +} + +export default resolveApiClient diff --git a/packages/sanity/src/_internal/cli/actions/backup/withRetry.ts b/packages/sanity/src/_internal/cli/actions/backup/withRetry.ts new file mode 100644 index 00000000000..b8502d5484a --- /dev/null +++ b/packages/sanity/src/_internal/cli/actions/backup/withRetry.ts @@ -0,0 +1,30 @@ +import debug from './debug' + +const MAX_RETRIES = 5 +const BACKOFF_DELAY_BASE = 200 + +const exponentialBackoff = (retryCount: number) => Math.pow(2, retryCount) * BACKOFF_DELAY_BASE + +async function withRetry( + operation: () => Promise, + maxRetries: number = MAX_RETRIES, +): Promise { + for (let retryCount = 0; retryCount < maxRetries; retryCount++) { + try { + return await operation() + } catch (err) { + // Immediately rethrow if the error is not server-related. + if (err.response && err.response.statusCode && err.response.statusCode < 500) { + throw err + } + + const retryDelay = exponentialBackoff(retryCount) + debug(`Error encountered, retrying after ${retryDelay}ms: %s`, err.message) + await new Promise((resolve) => setTimeout(resolve, retryDelay)) + } + } + + throw new Error('Operation failed after all retries') +} + +export default withRetry diff --git a/packages/sanity/src/_internal/cli/commands/backup/backupGroup.ts b/packages/sanity/src/_internal/cli/commands/backup/backupGroup.ts new file mode 100644 index 00000000000..e506e3b78af --- /dev/null +++ b/packages/sanity/src/_internal/cli/commands/backup/backupGroup.ts @@ -0,0 +1,16 @@ +import {type CliCommandGroupDefinition} from '@sanity/cli' + +// defaultApiVersion is the backend API version used for dataset backup. +// First version of the backup API is vX since this feature is not yet released +// and formal API documentation is pending. +export const defaultApiVersion = 'vX' + +const datasetBackupGroup: CliCommandGroupDefinition = { + name: 'backup', + signature: '[COMMAND]', + description: 'Manage dataset backups.', + isGroupRoot: true, + hideFromHelp: true, +} + +export default datasetBackupGroup diff --git a/packages/sanity/src/_internal/cli/commands/backup/disableBackupCommand.ts b/packages/sanity/src/_internal/cli/commands/backup/disableBackupCommand.ts new file mode 100644 index 00000000000..9f2fac3eb4a --- /dev/null +++ b/packages/sanity/src/_internal/cli/commands/backup/disableBackupCommand.ts @@ -0,0 +1,45 @@ +import {type CliCommandDefinition} from '@sanity/cli' + +import resolveApiClient from '../../actions/backup/resolveApiClient' +import {defaultApiVersion} from './backupGroup' + +const helpText = ` +Examples + sanity backup disable DATASET_NAME +` + +const disableDatasetBackupCommand: CliCommandDefinition = { + name: 'disable', + group: 'backup', + signature: '[DATASET_NAME]', + description: 'Disable backup for a dataset.', + helpText, + action: async (args, context) => { + const {output, chalk} = context + const [dataset] = args.argsWithoutOptions + const {projectId, datasetName, token, client} = await resolveApiClient( + context, + dataset, + defaultApiVersion, + ) + + try { + await client.request({ + method: 'PUT', + headers: {Authorization: `Bearer ${token}`}, + uri: `/projects/${projectId}/datasets/${datasetName}/settings/backups`, + body: { + enabled: false, + }, + }) + output.print(`${chalk.green(`Disabled daily backups for dataset ${datasetName}\n`)}`) + } catch (error) { + const msg = error.statusCode + ? error.response.body.message + : error.message || error.statusMessage + output.print(`${chalk.red(`Disabling dataset backup failed: ${msg}`)}\n`) + } + }, +} + +export default disableDatasetBackupCommand diff --git a/packages/sanity/src/_internal/cli/commands/backup/downloadBackupCommand.ts b/packages/sanity/src/_internal/cli/commands/backup/downloadBackupCommand.ts new file mode 100644 index 00000000000..46c21ca5a97 --- /dev/null +++ b/packages/sanity/src/_internal/cli/commands/backup/downloadBackupCommand.ts @@ -0,0 +1,282 @@ +import {createWriteStream, existsSync, mkdirSync} from 'node:fs' +import {mkdtemp} from 'node:fs/promises' +import {tmpdir} from 'node:os' +import path from 'node:path' + +import { + type CliCommandArguments, + type CliCommandContext, + type CliCommandDefinition, + type SanityClient, +} from '@sanity/cli' +import {absolutify} from '@sanity/util/fs' +import {Mutex} from 'async-mutex' +import createDebug from 'debug' +import {isString} from 'lodash' +import prettyMs from 'pretty-ms' +import {hideBin} from 'yargs/helpers' +import yargs from 'yargs/yargs' + +import archiveDir from '../../actions/backup/archiveDir' +import chooseBackupIdPrompt from '../../actions/backup/chooseBackupIdPrompt' +import cleanupTmpDir from '../../actions/backup/cleanupTmpDir' +import downloadAsset from '../../actions/backup/downloadAsset' +import downloadDocument from '../../actions/backup/downloadDocument' +import {type File, PaginatedGetBackupStream} from '../../actions/backup/fetchNextBackupPage' +import parseApiErr from '../../actions/backup/parseApiErr' +import newProgress from '../../actions/backup/progressSpinner' +import resolveApiClient from '../../actions/backup/resolveApiClient' +import humanFileSize from '../../util/humanFileSize' +import isPathDirName from '../../util/isPathDirName' +import {defaultApiVersion} from './backupGroup' + +const debug = createDebug('sanity:backup') + +const DEFAULT_DOWNLOAD_CONCURRENCY = 10 +const MAX_DOWNLOAD_CONCURRENCY = 24 + +interface DownloadBackupOptions { + projectId: string + datasetName: string + token: string + backupId: string + outDir: string + outFileName: string + overwrite: boolean + concurrency: number +} + +const helpText = ` +Options + --backup-id The backup ID to download. (required) + --out The file or directory path the backup should download to. + --overwrite Allows overwriting of existing backup file. + --concurrency Concurrent number of backup item downloads. (max: 24) + +Examples + sanity backup download DATASET_NAME --backup-id 2024-01-01-backup-1 + sanity backup download DATASET_NAME --backup-id 2024-01-01-backup-2 --out /path/to/file + sanity backup download DATASET_NAME --backup-id 2024-01-01-backup-3 --out /path/to/file --overwrite +` + +function parseCliFlags(args: {argv?: string[]}) { + return yargs(hideBin(args.argv || process.argv).slice(2)) + .options('backup-id', {type: 'string'}) + .options('out', {type: 'string'}) + .options('concurrency', {type: 'number', default: DEFAULT_DOWNLOAD_CONCURRENCY}) + .options('overwrite', {type: 'boolean', default: false}).argv +} + +const downloadBackupCommand: CliCommandDefinition = { + name: 'download', + group: 'backup', + signature: '[DATASET_NAME]', + description: 'Download a dataset backup to a local file.', + helpText, + // eslint-disable-next-line max-statements + action: async (args, context) => { + const {output, chalk} = context + const [client, opts] = await prepareBackupOptions(context, args) + const {projectId, datasetName, backupId, outDir, outFileName} = opts + + // If any of the output path or file name is empty, cancel the operation. + if (outDir === '' || outFileName === '') { + output.print('Operation cancelled.') + return + } + const outFilePath = path.join(outDir, outFileName) + + output.print('╭───────────────────────────────────────────────────────────╮') + output.print('│ │') + output.print('│ Downloading backup for: │') + output.print(`│ ${chalk.bold('projectId')}: ${chalk.cyan(projectId).padEnd(56)} │`) + output.print(`│ ${chalk.bold('dataset')}: ${chalk.cyan(datasetName).padEnd(58)} │`) + output.print(`│ ${chalk.bold('backupId')}: ${chalk.cyan(backupId).padEnd(56)} │`) + output.print('│ │') + output.print('╰───────────────────────────────────────────────────────────╯') + output.print('') + output.print(`Downloading backup to "${chalk.cyan(outFilePath)}"`) + + const start = Date.now() + const progressSpinner = newProgress(output, 'Setting up backup environment...') + + // Create a unique temporary directory to store files before bundling them into the archive at outputPath. + // Temporary directories are normally deleted at the end of backup process, any unexpected exit may leave them + // behind, hence it is important to create a unique directory for each attempt. + const tmpOutDir = await mkdtemp(path.join(tmpdir(), `sanity-backup-`)) + + // Create required directories if they don't exist. + for (const dir of [outDir, path.join(tmpOutDir, 'images'), path.join(tmpOutDir, 'files')]) { + mkdirSync(dir, {recursive: true}) + } + + debug('Writing to temporary directory %s', tmpOutDir) + const tmpOutDocumentsFile = path.join(tmpOutDir, 'data.ndjson') + + // Handle concurrent writes to the same file using mutex. + const docOutStream = createWriteStream(tmpOutDocumentsFile) + const docWriteMutex = new Mutex() + + try { + const backupFileStream = new PaginatedGetBackupStream( + client, + opts.projectId, + opts.datasetName, + opts.backupId, + opts.token, + ) + + const files: File[] = [] + let i = 0 + for await (const file of backupFileStream) { + files.push(file) + i++ + progressSpinner.set({ + step: `Reading backup files...`, + update: true, + current: i, + total: backupFileStream.totalFiles, + }) + } + + let totalItemsDownloaded = 0 + // This is dynamically imported because this module is ESM only and this file gets compiled to CJS at this time. + const {default: pMap} = await import('p-map') + await pMap( + files, + async (file: File) => { + if (file.type === 'file' || file.type === 'image') { + await downloadAsset(file.url, file.name, file.type, tmpOutDir) + } else { + const doc = await downloadDocument(file.url) + await docWriteMutex.runExclusive(() => { + docOutStream.write(`${doc}\n`) + }) + } + + totalItemsDownloaded += 1 + progressSpinner.set({ + step: `Downloading documents and assets...`, + update: true, + current: totalItemsDownloaded, + total: backupFileStream.totalFiles, + }) + }, + {concurrency: opts.concurrency}, + ) + } catch (error) { + progressSpinner.fail() + const {message} = parseApiErr(error) + throw new Error(`Downloading dataset backup failed: ${message}`) + } + + progressSpinner.set({step: `Archiving files into a tarball...`, update: true}) + try { + await archiveDir(tmpOutDir, outFilePath, (processedBytes: number) => { + progressSpinner.update({ + step: `Archiving files into a tarball, ${humanFileSize(processedBytes)} bytes written...`, + }) + }) + } catch (err) { + progressSpinner.fail() + throw new Error(`Archiving backup failed: ${err.message}`) + } + + progressSpinner.set({ + step: `Cleaning up temporary files at ${chalk.cyan(`${tmpOutDir}`)}`, + }) + cleanupTmpDir(tmpOutDir) + + progressSpinner.set({ + step: `Backup download complete [${prettyMs(Date.now() - start)}]`, + }) + progressSpinner.succeed() + }, +} + +// prepareBackupOptions validates backup options from CLI and prepares Client and DownloadBackupOptions. +async function prepareBackupOptions( + context: CliCommandContext, + args: CliCommandArguments, +): Promise<[SanityClient, DownloadBackupOptions]> { + const flags = await parseCliFlags(args) + const [dataset] = args.argsWithoutOptions + const {prompt, workDir} = context + const {projectId, datasetName, client} = await resolveApiClient( + context, + dataset, + defaultApiVersion, + ) + + const {token} = client.config() + if (!isString(token) || token.length < 1) { + throw new Error(`token is missing`) + } + + if (!isString(datasetName) || datasetName.length < 1) { + throw new Error(`dataset ${datasetName} must be a valid dataset name`) + } + + const backupId = String(flags['backup-id'] || (await chooseBackupIdPrompt(context, datasetName))) + if (backupId.length < 1) { + throw new Error(`backup-id ${flags['backup-id']} should be a valid string`) + } + + if ('concurrency' in flags) { + if (flags.concurrency < 1 || flags.concurrency > MAX_DOWNLOAD_CONCURRENCY) { + throw new Error(`concurrency should be in 1 to ${MAX_DOWNLOAD_CONCURRENCY} range`) + } + } + + const defaultOutFileName = `${datasetName}-backup-${backupId}.tar.gz` + let out = await (async (): Promise => { + if (flags.out !== undefined) { + // Rewrite the output path to an absolute path, if it is not already. + return absolutify(flags.out) + } + + const input = await prompt.single({ + type: 'input', + message: 'Output path:', + default: path.join(workDir, defaultOutFileName), + filter: absolutify, + }) + return input + })() + + // If path is a directory name, then add a default file name to the path. + if (isPathDirName(out)) { + out = path.join(out, defaultOutFileName) + } + + // If the file already exists, ask for confirmation if it should be overwritten. + if (!flags.overwrite && existsSync(out)) { + const shouldOverwrite = await prompt.single({ + type: 'confirm', + message: `File "${out}" already exists, would you like to overwrite it?`, + default: false, + }) + + // If the user does not want to overwrite the file, set the output path to an empty string. + // This should be handled by the caller of this function as cancel operation. + if (!shouldOverwrite) { + out = '' + } + } + + return [ + client, + { + projectId, + datasetName, + backupId, + token, + outDir: path.dirname(out), + outFileName: path.basename(out), + overwrite: flags.overwrite, + concurrency: flags.concurrency || DEFAULT_DOWNLOAD_CONCURRENCY, + }, + ] +} + +export default downloadBackupCommand diff --git a/packages/sanity/src/_internal/cli/commands/backup/enableBackupCommand.ts b/packages/sanity/src/_internal/cli/commands/backup/enableBackupCommand.ts new file mode 100644 index 00000000000..48caa4a2286 --- /dev/null +++ b/packages/sanity/src/_internal/cli/commands/backup/enableBackupCommand.ts @@ -0,0 +1,49 @@ +import {type CliCommandDefinition} from '@sanity/cli' + +import resolveApiClient from '../../actions/backup/resolveApiClient' +import {defaultApiVersion} from './backupGroup' + +const helpText = ` +Examples + sanity backup enable DATASET_NAME +` + +const enableDatasetBackupCommand: CliCommandDefinition = { + name: 'enable', + group: 'backup', + signature: '[DATASET_NAME]', + description: 'Enable backup for a dataset.', + helpText, + action: async (args, context) => { + const {output, chalk} = context + const [dataset] = args.argsWithoutOptions + const {projectId, datasetName, token, client} = await resolveApiClient( + context, + dataset, + defaultApiVersion, + ) + + try { + await client.request({ + method: 'PUT', + headers: {Authorization: `Bearer ${token}`}, + uri: `/projects/${projectId}/datasets/${datasetName}/settings/backups`, + body: { + enabled: true, + }, + }) + + output.print(`${chalk.green(`Enabled backups for dataset ${datasetName}.\n`)}`) + + output.print( + `${chalk.bold(`Retention policies may apply depending on your plan and agreement.\n`)}`, + ) + } catch (error) { + const msg = error.statusCode + ? error.response.body.message + : error.message || error.statusMessage + output.print(`${chalk.red(`Enabling dataset backup failed: ${msg}`)}\n`) + } + }, +} +export default enableDatasetBackupCommand diff --git a/packages/sanity/src/_internal/cli/commands/backup/listBackupCommand.ts b/packages/sanity/src/_internal/cli/commands/backup/listBackupCommand.ts new file mode 100644 index 00000000000..17e4154ed7c --- /dev/null +++ b/packages/sanity/src/_internal/cli/commands/backup/listBackupCommand.ts @@ -0,0 +1,149 @@ +import {type CliCommandDefinition} from '@sanity/cli' +import {Table} from 'console-table-printer' +import {isAfter, isValid, lightFormat, parse} from 'date-fns' +import {hideBin} from 'yargs/helpers' +import yargs from 'yargs/yargs' + +import parseApiErr from '../../actions/backup/parseApiErr' +import resolveApiClient from '../../actions/backup/resolveApiClient' +import {defaultApiVersion} from './backupGroup' + +const DEFAULT_LIST_BACKUP_LIMIT = 30 + +interface ListDatasetBackupFlags { + before?: string + after?: string + limit?: string +} + +type ListBackupRequestQueryParams = { + before?: string + after?: string + limit: string +} + +type ListBackupResponse = { + backups: ListBackupResponseItem[] +} + +type ListBackupResponseItem = { + id: string + createdAt: string +} + +const helpText = ` +Options + --limit Maximum number of backups returned. Default 30. + --after Only return backups after this date (inclusive) + --before Only return backups before this date (exclusive). Cannot be younger than if specified. + +Examples + sanity backup list DATASET_NAME + sanity backup list DATASET_NAME --limit 50 + sanity backup list DATASET_NAME --after 2024-01-31 --limit 10 + sanity backup list DATASET_NAME --after 2024-01-31 --before 2024-01-10 +` + +function parseCliFlags(args: {argv?: string[]}) { + return yargs(hideBin(args.argv || process.argv).slice(2)) + .options('after', {type: 'string'}) + .options('before', {type: 'string'}) + .options('limit', {type: 'number', default: DEFAULT_LIST_BACKUP_LIMIT, alias: 'l'}).argv +} + +const listDatasetBackupCommand: CliCommandDefinition = { + name: 'list', + group: 'backup', + signature: '[DATASET_NAME]', + description: 'List available backups for a dataset.', + helpText, + action: async (args, context) => { + const {output, chalk} = context + const flags = await parseCliFlags(args) + const [dataset] = args.argsWithoutOptions + + const {projectId, datasetName, token, client} = await resolveApiClient( + context, + dataset, + defaultApiVersion, + ) + + const query: ListBackupRequestQueryParams = {limit: DEFAULT_LIST_BACKUP_LIMIT.toString()} + if (flags.limit) { + // We allow limit up to Number.MAX_SAFE_INTEGER to leave it for server-side validation, + // while still sending sensible value in limit string. + if (flags.limit < 1 || flags.limit > Number.MAX_SAFE_INTEGER) { + throw new Error( + `Parsing --limit: must be an integer between 1 and ${Number.MAX_SAFE_INTEGER}`, + ) + } + query.limit = flags.limit.toString() + } + + if (flags.before || flags.after) { + try { + const parsedBefore = processDateFlags(flags.before) + const parsedAfter = processDateFlags(flags.after) + + if (parsedAfter && parsedBefore && isAfter(parsedAfter, parsedBefore)) { + throw new Error('--after date must be before --before') + } + + query.before = flags.before + query.after = flags.after + } catch (err) { + throw new Error(`Parsing date flags: ${err}`) + } + } + + let response + try { + response = await client.request({ + headers: {Authorization: `Bearer ${token}`}, + uri: `/projects/${projectId}/datasets/${datasetName}/backups`, + query: {...query}, + }) + } catch (error) { + const {message} = parseApiErr(error) + output.error(`${chalk.red(`List dataset backup failed: ${message}`)}\n`) + } + + if (response && response.backups) { + if (response.backups.length === 0) { + output.print('No backups found.') + return + } + + const table = new Table({ + columns: [ + {name: 'resource', title: 'RESOURCE', alignment: 'left'}, + {name: 'createdAt', title: 'CREATED AT', alignment: 'left'}, + {name: 'backupId', title: 'BACKUP ID', alignment: 'left'}, + ], + }) + + response.backups.forEach((backup: ListBackupResponseItem) => { + const {id, createdAt} = backup + table.addRow({ + resource: 'Dataset', + createdAt: lightFormat(Date.parse(createdAt), 'yyyy-MM-dd HH:mm:ss'), + backupId: id, + }) + }) + + table.printTable() + } + }, +} + +function processDateFlags(date: string | undefined): Date | undefined { + if (!date) return undefined + const parsedDate = parse(date, 'yyyy-MM-dd', new Date()) + if (isValid(parsedDate)) { + return parsedDate + } + + throw new Error(`Invalid ${date} date format. Use YYYY-MM-DD`) +} + +export default listDatasetBackupCommand diff --git a/packages/sanity/src/_internal/cli/commands/index.ts b/packages/sanity/src/_internal/cli/commands/index.ts index 32877e16a68..37463949ef9 100644 --- a/packages/sanity/src/_internal/cli/commands/index.ts +++ b/packages/sanity/src/_internal/cli/commands/index.ts @@ -1,5 +1,10 @@ import {type CliCommandDefinition, type CliCommandGroupDefinition} from '@sanity/cli' +import backupGroup from './backup/backupGroup' +import disableBackupCommand from './backup/disableBackupCommand' +import downloadBackupCommand from './backup/downloadBackupCommand' +import enableBackupCommand from './backup/enableBackupCommand' +import listBackupCommand from './backup/listBackupCommand' import buildCommand from './build/buildCommand' import checkCommand from './check/checkCommand' import configCheckCommand from './config/configCheckCommand' @@ -64,6 +69,11 @@ const commands: (CliCommandDefinition | CliCommandGroupDefinition)[] = [ deleteDatasetCommand, copyDatasetCommand, aliasDatasetCommand, + backupGroup, + listBackupCommand, + downloadBackupCommand, + disableBackupCommand, + enableBackupCommand, corsGroup, listCorsOriginsCommand, addCorsOriginCommand, diff --git a/packages/sanity/src/_internal/cli/util/humanFileSize.ts b/packages/sanity/src/_internal/cli/util/humanFileSize.ts new file mode 100644 index 00000000000..d5d6f95bb53 --- /dev/null +++ b/packages/sanity/src/_internal/cli/util/humanFileSize.ts @@ -0,0 +1,6 @@ +function humanFileSize(size: number): string { + const i = size == 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024)) + return `${(size / Math.pow(1024, i)).toFixed(2)} ${['B', 'kB', 'MB', 'GB', 'TB'][i]}` +} + +export default humanFileSize diff --git a/packages/sanity/src/_internal/cli/util/isPathDirName.ts b/packages/sanity/src/_internal/cli/util/isPathDirName.ts new file mode 100644 index 00000000000..342c0e5578a --- /dev/null +++ b/packages/sanity/src/_internal/cli/util/isPathDirName.ts @@ -0,0 +1,6 @@ +function isPathDirName(filepath: string): boolean { + // Check if the path has an extension, commonly indicating a file + return !/\.\w+$/.test(filepath) +} + +export default isPathDirName diff --git a/yarn.lock b/yarn.lock index 8769b527cfd..9b298c9e79f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4087,6 +4087,13 @@ "@turf/helpers" "^5.1.5" "@turf/meta" "^5.1.5" +"@types/archiver@^6.0.2": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2" + integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw== + dependencies: + "@types/readdir-glob" "*" + "@types/argparse@1.0.38": version "1.0.38" resolved "https://registry.yarnpkg.com/@types/argparse/-/argparse-1.0.38.tgz#a81fd8606d481f873a3800c6ebae4f1d768a56a9" @@ -4486,6 +4493,13 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/readdir-glob@*": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@types/readdir-glob/-/readdir-glob-1.1.5.tgz#21a4a98898fc606cb568ad815f2a0eedc24d412a" + integrity sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg== + dependencies: + "@types/node" "*" + "@types/refractor@^3.0.0": version "3.0.2" resolved "https://registry.yarnpkg.com/@types/refractor/-/refractor-3.0.2.tgz#2d42128d59f78f84d2c799ffc5ab5cadbcba2d82" @@ -5108,6 +5122,18 @@ archiver-utils@^2.1.0: normalize-path "^3.0.0" readable-stream "^2.0.0" +archiver-utils@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-4.0.1.tgz#66ad15256e69589a77f706c90c6dbcc1b2775d2a" + integrity sha512-Q4Q99idbvzmgCTEAAhi32BkOyq8iVI5EwdO0PmBDSGIzzjYNdcFn7Q7k3OzbLy4kLUPXfJtG6fO2RjftXbobBg== + dependencies: + glob "^8.0.0" + graceful-fs "^4.2.0" + lazystream "^1.0.0" + lodash "^4.17.15" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + archiver@^5.0.0: version "5.3.1" resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.1.tgz#21e92811d6f09ecfce649fbefefe8c79e57cbbb6" @@ -5121,6 +5147,19 @@ archiver@^5.0.0: tar-stream "^2.2.0" zip-stream "^4.1.0" +archiver@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/archiver/-/archiver-6.0.1.tgz#d56968d4c09df309435adb5a1bbfc370dae48133" + integrity sha512-CXGy4poOLBKptiZH//VlWdFuUC1RESbdZjGjILwBuZ73P7WkAUN0htfSfBq/7k6FRFlpu7bg4JOkj1vU9G6jcQ== + dependencies: + archiver-utils "^4.0.1" + async "^3.2.4" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.1.2" + tar-stream "^3.0.0" + zip-stream "^5.0.1" + are-we-there-yet@~1.1.2: version "1.1.7" resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz#b15474a932adab4ff8a50d9adfa7e4e926f21146" @@ -5367,11 +5406,23 @@ async-each@^1.0.0: resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.6.tgz#52f1d9403818c179b7561e11a5d1b77eb2160e77" integrity sha512-c646jH1avxr+aVpndVMeAfYw7wAa6idufrlN3LPA4PmKS0QEGp6PIC9nwz0WQkkvBGAMEki3pFdtxaF39J9vvg== +async-mutex@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.4.1.tgz#bccf55b96f2baf8df90ed798cb5544a1f6ee4c2c" + integrity sha512-WfoBo4E/TbCX1G95XTjbWTE3X2XLG0m1Xbv2cwOtuPdyH9CZvnaA5nCt1ucjaKEgW2A5IF71hxrRhr83Je5xjA== + dependencies: + tslib "^2.4.0" + async@^3.2.3: version "3.2.4" resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== +async@^3.2.4: + version "3.2.5" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" + integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== + asynciterator.prototype@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz#8c5df0514936cdd133604dfcc9d3fb93f09b2b62" @@ -6322,6 +6373,16 @@ compress-commons@^4.1.0: normalize-path "^3.0.0" readable-stream "^3.6.0" +compress-commons@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-5.0.1.tgz#e46723ebbab41b50309b27a0e0f6f3baed2d6590" + integrity sha512-MPh//1cERdLtqwO3pOFLeXtpuai0Y2WCd5AhtKxznqM7WtaMYaOEMSgn45d9D10sIHSfIKE603HlOp8OPGrvag== + dependencies: + crc-32 "^1.2.0" + crc32-stream "^5.0.0" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + compute-scroll-into-view@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/compute-scroll-into-view/-/compute-scroll-into-view-3.0.3.tgz#c418900a5c56e2b04b885b54995df164535962b1" @@ -6581,6 +6642,14 @@ crc32-stream@^4.0.2: crc-32 "^1.2.0" readable-stream "^3.4.0" +crc32-stream@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-5.0.0.tgz#a97d3a802c8687f101c27cc17ca5253327354720" + integrity sha512-B0EPa1UK+qnpBZpG+7FgPCu0J2ETLpXq09o9BkLkEAhdB6Z61Qo4pJ3JYu0c+Qi+/SAL7QThqnzS06pmSSyZaw== + dependencies: + crc-32 "^1.2.0" + readable-stream "^3.4.0" + create-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320" @@ -8902,7 +8971,7 @@ glob@^7.0.5, glob@^7.1.1, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^8.0.3: +glob@^8.0.0, glob@^8.0.3: version "8.1.0" resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== @@ -13708,7 +13777,7 @@ readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stre isarray "0.0.1" string_decoder "~0.10.x" -readdir-glob@^1.0.0: +readdir-glob@^1.0.0, readdir-glob@^1.1.2: version "1.1.3" resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.3.tgz#c3d831f51f5e7bfa62fa2ffbe4b508c640f09584" integrity sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA== @@ -15206,7 +15275,7 @@ tar-stream@^2.1.4, tar-stream@^2.2.0: inherits "^2.0.3" readable-stream "^3.1.1" -tar-stream@^3.1.7: +tar-stream@^3.0.0, tar-stream@^3.1.7: version "3.1.7" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b" integrity sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ== @@ -16585,6 +16654,15 @@ zip-stream@^4.1.0: compress-commons "^4.1.0" readable-stream "^3.6.0" +zip-stream@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-5.0.1.tgz#cf3293bba121cad98be2ec7f05991d81d9f18134" + integrity sha512-UfZ0oa0C8LI58wJ+moL46BDIMgCQbnsb+2PoiJYtonhBsMh2bq1eRBVkvjfVsqbEHd9/EgKPUuL9saSSsec8OA== + dependencies: + archiver-utils "^4.0.1" + compress-commons "^5.0.1" + readable-stream "^3.6.0" + zod@^3.22.4: version "3.22.4" resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff"