Skip to content

Commit

Permalink
feat(cli): allow user provided concurrency
Browse files Browse the repository at this point in the history
  • Loading branch information
bjoerge committed Jan 26, 2024
1 parent cd138e4 commit 3a431df
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 7 deletions.
1 change: 1 addition & 0 deletions packages/@sanity/migrate/src/_exports/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ export * from '../it-utils'
export * from '../runner/run'
export * from '../runner/dryRun'
export * from '../runner/collectMigrationMutations'
export {MAX_MUTATION_CONCURRENCY, DEFAULT_MUTATION_CONCURRENCY} from '../runner/constants'
1 change: 1 addition & 0 deletions packages/@sanity/migrate/src/runner/constants.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
export const MUTATION_ENDPOINT_MAX_BODY_SIZE = 1024 * 256 // 256KB
export const DEFAULT_MUTATION_CONCURRENCY = 6
export const MAX_MUTATION_CONCURRENCY = 10
15 changes: 10 additions & 5 deletions packages/@sanity/migrate/src/runner/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ import {toFetchOptions} from '../fetch-utils/sanityRequestOptions'
import {commitMutations} from '../destinations/commitMutations'
import {collectMigrationMutations} from './collectMigrationMutations'
import {batchMutations} from './utils/batchMutations'
import {DEFAULT_MUTATION_CONCURRENCY, MUTATION_ENDPOINT_MAX_BODY_SIZE} from './constants'
import {
DEFAULT_MUTATION_CONCURRENCY,
MAX_MUTATION_CONCURRENCY,
MUTATION_ENDPOINT_MAX_BODY_SIZE,
} from './constants'
import {toSanityMutations} from './utils/toSanityMutations'

interface MigrationRunnerOptions {
Expand Down Expand Up @@ -39,10 +43,11 @@ export async function* run(config: MigrationRunnerOptions, migration: Migration)
}),
)

const concurrency = Math.min(
DEFAULT_MUTATION_CONCURRENCY,
config?.concurrency ?? DEFAULT_MUTATION_CONCURRENCY,
)
const concurrency = config?.concurrency ?? DEFAULT_MUTATION_CONCURRENCY

if (concurrency > MAX_MUTATION_CONCURRENCY) {
throw new Error(`Concurrency exceeds maximum allowed value (${MAX_MUTATION_CONCURRENCY})`)
}

const batches = batchMutations(toSanityMutations(mutations), MUTATION_ENDPOINT_MAX_BODY_SIZE)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import {
fromExportArchive,
fromExportEndpoint,
safeJsonParser,
MAX_MUTATION_CONCURRENCY,
DEFAULT_MUTATION_CONCURRENCY,
Migration,
ndjson,
run,
Expand All @@ -16,7 +18,10 @@ import {format} from './mutationFormatter'

const helpText = `
Options
--dryRun <boolean> Whether or not to dry run the migration. Default to true, to actually run the migration this has to be set to false
--dry <boolean> Whether or not to dry run the migration. Default to true, to actually run the migration this has to be set to false
--from-export <export.tar.gz> Use a local dataset export as source for migration instead of calling the Sanity API. Note: this is only supported for dry runs.
--concurrency <concurrent> How many mutation requests to run in parallel. Must be between 1 and ${MAX_MUTATION_CONCURRENCY}. Default: ${DEFAULT_MUTATION_CONCURRENCY}.
Examples
sanity migration run
Expand All @@ -27,6 +32,7 @@ Examples
interface CreateFlags {
dry?: 'true' | 'false' | 'yes' | 'no'
'from-export'?: string
concurrency?: number
}

const tryExtensions = ['mjs', 'js', 'ts', 'cjs']
Expand Down Expand Up @@ -112,6 +118,19 @@ const createMigrationCommand: CliCommandDefinition<CreateFlags> = {
throw new Error('Can only dry run migrations from a dataset export file')
}

const concurrency = args.extOptions.concurrency
if (concurrency !== undefined) {
if (concurrency > MAX_MUTATION_CONCURRENCY) {
throw new Error(
`Concurrency exceeds the maximum allowed value of ${MAX_MUTATION_CONCURRENCY}`,
)
}

if (concurrency === 0) {
throw new Error(`Concurrency must be a positive number, got ${concurrency}`)
}
}

const {dataset, projectId, apiHost, token} = apiClient({
requireUser: true,
requireProject: true,
Expand All @@ -127,7 +146,7 @@ const createMigrationCommand: CliCommandDefinition<CreateFlags> = {

const progress = dry
? dryRun({api: apiConfig}, migration, context)
: run({api: apiConfig}, migration)
: run({api: apiConfig, concurrency}, migration)

for await (const result of progress) {
output.print(result)
Expand Down

0 comments on commit 3a431df

Please sign in to comment.