feat(migration): support passing an array of documentTypes to migrati…

…on (#5566)
sanity-io · Jan 30, 2024 · 458ac49 · 458ac49
1 parent 4ab2ac2
commit 458ac49
Show file tree

Hide file tree

Showing 11 changed files with 41 additions and 23 deletions.
diff --git a/dev/test-studio/migrations/cleanup-empty/cleanup-empty-values.ts b/dev/test-studio/migrations/cleanup-empty/cleanup-empty-values.ts
@@ -4,7 +4,7 @@ import {unset} from 'sanity/migrate/mutations'
 
 export default defineMigration({
   name: 'Cleanup empty values',
-  documentType: 'species',
+  documentTypes: ['playlist', 'species'],
   migrate: {
     object(node) {
       if (Object.keys(node).filter((k) => !k.startsWith('_')).length) {

diff --git a/dev/test-studio/migrations/convert-string-to-pt/convert-string-to-pte.ts b/dev/test-studio/migrations/convert-string-to-pt/convert-string-to-pte.ts
@@ -5,7 +5,7 @@ import {set} from 'sanity/migrate/mutations'
 
 export default defineMigration({
   name: 'Convert string to PortableText at `some.path` in documents of type `someType`',
-  documentType: 'someType',
+  documentTypes: ['someType'],
   migrate: {
     string(node, path, ctx) {
       if (isEqual(path, ['some', 'path'])) {

diff --git a/dev/test-studio/migrations/rename-location-to-address/index.ts b/dev/test-studio/migrations/rename-location-to-address/index.ts
@@ -12,7 +12,7 @@ const addresses = [
 
 export default defineMigration({
   name: 'Rename Location to Address',
-  documentType: 'author',
+  documentTypes: ['author'],
   migrate: {
     document(doc) {
       return patch(doc._id, [

diff --git a/dev/test-studio/migrations/sync-from-spreadsheet/sync.ts b/dev/test-studio/migrations/sync-from-spreadsheet/sync.ts
@@ -6,7 +6,7 @@ declare function fetchGoogleSpreadSheet(id: string): AsyncIterable<Record<string
 // defineSync?
 export default defineMigration({
   name: 'Sync from some spreadsheet somewhere',
-  documentType: 'someType',
+  documentTypes: ['someType'],
   async *migrate() {
     for await (const row of fetchGoogleSpreadSheet('some-spreadsheet-id')) {
       yield createIfNotExists({_type: 'someType', _id: row.id, name: row.name})

diff --git a/packages/@sanity/migrate/src/fetch-utils/endpoints.ts b/packages/@sanity/migrate/src/fetch-utils/endpoints.ts
@@ -22,11 +22,11 @@ export const endpoints = {
       path: `/query/${dataset}`,
       searchParams: [],
     }),
-    export: (dataset: string): Endpoint => ({
+    export: (dataset: string, documentTypes: string): Endpoint => ({
       global: false,
       method: 'GET',
       path: `/data/export/${dataset}`,
-      searchParams: [],
+      searchParams: [['types', documentTypes]],
     }),
     mutate: (
       dataset: string,

diff --git a/packages/@sanity/migrate/src/runner/dryRun.ts b/packages/@sanity/migrate/src/runner/dryRun.ts
@@ -12,9 +12,12 @@ interface MigrationRunnerOptions {
 export async function* dryRun(config: MigrationRunnerOptions, migration: Migration) {
   const mutations = collectMigrationMutations(
     migration,
-    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {
-      parse: safeJsonParser,
-    }),
+    ndjson<SanityDocument>(
+      await fromExportEndpoint({...config.api, documentTypes: migration.documentTypes}),
+      {
+        parse: safeJsonParser,
+      },
+    ),
   )
 
   for await (const mutation of mutations) {

diff --git a/packages/@sanity/migrate/src/runner/normalizeMigrateDefinition.ts b/packages/@sanity/migrate/src/runner/normalizeMigrateDefinition.ts
@@ -13,7 +13,7 @@ export function normalizeMigrateDefinition(migration: Migration): AsyncIterableM
   }
   return createAsyncIterableMutation(migration.migrate, {
     filter: migration.filter,
-    documentType: migration.documentType,
+    documentTypes: migration.documentTypes,
   })
 }
 
@@ -38,11 +38,14 @@ function isOperation(value: Mutation | NodePatch | Operation): value is Operatio
 
 export function createAsyncIterableMutation(
   migration: NodeMigration,
-  opts: {filter?: string; documentType?: string},
+  opts: {filter?: string; documentTypes?: string[]},
 ): AsyncIterableMigration {
+  const documentTypesSet = new Set(opts.documentTypes)
+
   return async function* run(docs, context) {
     for await (const doc of docs) {
-      if (doc._type !== opts.documentType) continue
+      if (!documentTypesSet.has(doc._type)) continue
+
       const documentMutations = collectDocumentMutations(migration, doc, context)
       if (documentMutations.length > 0) {
         yield documentMutations

diff --git a/packages/@sanity/migrate/src/runner/run.ts b/packages/@sanity/migrate/src/runner/run.ts
@@ -62,7 +62,10 @@ export async function run(config: MigrationRunnerOptions, migration: Migration)
     currentMutations: [],
   }
   const documents = tap(
-    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {parse: safeJsonParser}),
+    ndjson<SanityDocument>(
+      await fromExportEndpoint({...config.api, documentTypes: migration.documentTypes}),
+      {parse: safeJsonParser},
+    ),
     () => {
       config.onProgress?.({...stats, documents: ++stats.documents})
     },

diff --git a/packages/@sanity/migrate/src/sources/fromExportEndpoint.ts b/packages/@sanity/migrate/src/sources/fromExportEndpoint.ts
@@ -1,18 +1,20 @@
 import {createSafeJsonParser} from '@sanity/util/createSafeJsonParser'
+import {SanityDocument} from '@sanity/types'
 import {fetchAsyncIterator} from '../fetch-utils/fetchStream'
 import {toFetchOptions} from '../fetch-utils/sanityRequestOptions'
 import {endpoints} from '../fetch-utils/endpoints'
-import {APIConfig} from '../types'
-import {SanityDocument} from '@sanity/types'
+import {ExportAPIConfig} from '../types'
 
-export function fromExportEndpoint(options: APIConfig) {
+export function fromExportEndpoint(
+  options: ExportAPIConfig,
+): Promise<AsyncGenerator<Uint8Array, void, unknown>> {
   return fetchAsyncIterator(
     toFetchOptions({
       projectId: options.projectId,
       apiVersion: options.apiVersion,
       token: options.token,
       apiHost: options.apiHost ?? 'api.sanity.io',
-      endpoint: endpoints.data.export(options.dataset),
+      endpoint: endpoints.data.export(options.dataset, options.documentTypes.join(',')),
     }),
   )
 }

diff --git a/packages/@sanity/migrate/src/types.ts b/packages/@sanity/migrate/src/types.ts
@@ -16,7 +16,7 @@ export interface Migration<Def extends MigrateDefinition = MigrateDefinition> {
    * Define input for the migration. If the migration uses an existing set of documents as starting point, define the filter here.
    */
   filter?: string
-  documentType: string
+  documentTypes: string[]
   migrate: Def
 }
 
@@ -34,6 +34,10 @@ export interface APIConfig {
   apiHost?: string
 }
 
+export interface ExportAPIConfig extends APIConfig {
+  documentTypes: string[]
+}
+
 export interface NodeMigrationContext {
   withDocument(id: string): Promise<SanityDocument | null>
 }

diff --git a/packages/sanity/src/_internal/cli/commands/migration/runMigrationCommand.ts b/packages/sanity/src/_internal/cli/commands/migration/runMigrationCommand.ts
@@ -189,7 +189,7 @@ const createMigrationCommand: CliCommandDefinition<CreateFlags> = {
 
   Project id:     ${chalk.bold(projectId)}
   Dataset:        ${chalk.bold(dataset)}
-  Document type:  ${chalk.bold(migration.documentType)}
+  Document type:  ${chalk.bold(migration.documentTypes.join(','))}
 
   ${progress.documents} documents processed…
   ${progress.mutations} mutations generated…
@@ -235,13 +235,16 @@ interface MigrationRunnerOptions {
 async function* dryRun(
   config: MigrationRunnerOptions,
   migration: Migration,
-  {output, chalk}: CliCommandContext,
+  {chalk}: CliCommandContext,
 ) {
   const mutations = collectMigrationMutations(
     migration,
-    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {
-      parse: safeJsonParser,
-    }),
+    ndjson<SanityDocument>(
+      await fromExportEndpoint({...config.api, documentTypes: migration.documentTypes}),
+      {
+        parse: safeJsonParser,
+      },
+    ),
   )
 
   for await (const mutation of mutations) {