feat(migrate): use safe JSON parser when streaming from Export HTTP A…

…PI (#5542)

* refactor(util): create shared JSON parser for `@sanity/export` and `@sanity/migrate`

* feat(migrate): add safe JSON parser

* chore(util): update comment

* chore(export): update comment

* feat(migrate): use safe JSON parser when streaming from Export HTTP API

* feat(migrate): add JSON options and type parameter to NDJSON utility

* refactor(migrate): use type parameter

* fix(migrate): add missing export

packages/@sanity/export/package.json

@@ -31,6 +31,7 @@
     "test": "jest"
   },
   "dependencies": {
+    "@sanity/util": "3.26.1",
     "archiver": "^5.0.0",
     "debug": "^3.2.7",
     "get-it": "^8.4.4",

packages/@sanity/export/src/tryParseJson.js

@@ -1,21 +1,13 @@
-module.exports = (line) => {
-  try {
-    return JSON.parse(line)
-  } catch (err) {
-    // Catch half-done lines with an error at the end
-    const errorPosition = line.lastIndexOf('{"error":')
-    if (errorPosition === -1) {
-      err.message = `${err.message} (${line})`
-      throw err
-    }
+const {createSafeJsonParser} = require('@sanity/util/createSafeJsonParser')
 
-    const errorJson = line.slice(errorPosition)
-    const errorLine = JSON.parse(errorJson)
-    const error = errorLine && errorLine.error
-    if (error && error.description) {
-      throw new Error(`Error streaming dataset: ${error.description}\n\n${errorJson}\n`)
-    }
-
-    throw err
-  }
-}
+/**
+ * Safe JSON parser that is able to handle lines interrupted by an error object.
+ *
+ * This may occur when streaming NDJSON from the Export HTTP API.
+ *
+ * @internal
+ * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
+ */
+module.exports = createSafeJsonParser({
+  errorLabel: 'Error streaming dataset',
+})

packages/@sanity/migrate/package.json

@@ -77,6 +77,7 @@
     "@bjoerge/mutiny": "^0.5.1",
     "@sanity/client": "^6.11.1",
     "@sanity/types": "^3.23.4",
+    "@sanity/util": "3.26.1",
     "arrify": "^2.0.1",
     "fast-fifo": "^1.3.2"
   },

packages/@sanity/migrate/src/_exports/index.ts

@@ -1,6 +1,7 @@
 export {fromExportArchive} from '../sources/fromExportArchive'
 export {fromExportEndpoint} from '../sources/fromExportEndpoint'
 export {fromDocuments} from '../sources/fromDocuments'
+export {safeJsonParser} from '../sources/fromExportEndpoint'
 export * from '../types'
 export * from '../defineMigration'
 export * from '../it-utils'

packages/@sanity/migrate/src/it-utils/__test__/json.test.ts

@@ -1,39 +1,31 @@
 import {parseJSON} from '../json'
-import {createSafeJsonParser} from '../createSafeJsonParser'
 
 test('parse JSON', async () => {
   const gen = async function* () {
     yield '{"someString": "string"}'
     yield '{"someNumber": 42}'
   }
 
-  const it = parseJSON(gen(), {})
+  const it = parseJSON(gen())
 
   expect(await it.next()).toEqual({value: {someString: 'string'}, done: false})
   expect(await it.next()).toEqual({value: {someNumber: 42}, done: false})
   expect(await it.next()).toEqual({value: undefined, done: true})
 })
 
-test('parse JSON with interrupting error', async () => {
+test('parse JSON with a custom parser', async () => {
   const gen = async function* () {
     yield '{"someString": "string"}'
-    yield '{"someString": "str{"error":{"description":"Some error"}}'
+    yield '{"someNumber": 42}'
   }
 
   const it = parseJSON(gen(), {
-    parse: createSafeJsonParser({
-      errorLabel: 'Error parsing JSON',
+    parse: (line) => ({
+      parsed: JSON.parse(line),
     }),
   })
 
-  expect(await it.next()).toEqual({value: {someString: 'string'}, done: false})
-
-  await expect(async () => {
-    await it.next()
-  }).rejects.toThrowErrorMatchingInlineSnapshot(`
-"Error parsing JSON: Some error
-
-{\\"error\\":{\\"description\\":\\"Some error\\"}}
-"
-`)
+  expect(await it.next()).toEqual({value: {parsed: {someString: 'string'}}, done: false})
+  expect(await it.next()).toEqual({value: {parsed: {someNumber: 42}}, done: false})
+  expect(await it.next()).toEqual({value: undefined, done: true})
 })

packages/@sanity/migrate/src/it-utils/json.ts

@@ -1,12 +1,12 @@
-type Parser<Type> = (line: string) => Type
+export type JSONParser<Type> = (line: string) => Type
 
-interface Options<Type> {
-  parse?: Parser<Type>
+export interface JSONOptions<Type> {
+  parse?: JSONParser<Type>
 }
 
 export async function* parseJSON<Type>(
   it: AsyncIterableIterator<string>,
-  {parse = JSON.parse}: Options<Type> = {},
+  {parse = JSON.parse}: JSONOptions<Type> = {},
 ): AsyncIterableIterator<Type> {
   for await (const chunk of it) {
     yield parse(chunk)

packages/@sanity/migrate/src/it-utils/ndjson.ts

@@ -1,8 +1,14 @@
 import {split} from './split'
 import {decodeText} from './decodeText'
-import {parseJSON} from './json'
+import {type JSONOptions, parseJSON} from './json'
 import {filter} from './filter'
 
-export function ndjson(it: AsyncIterableIterator<Uint8Array>) {
-  return parseJSON(filter(split(decodeText(it), '\n'), (line) => Boolean(line && line.trim())))
+export function ndjson<Type>(
+  it: AsyncIterableIterator<Uint8Array>,
+  options?: JSONOptions<Type>,
+): AsyncIterableIterator<Type> {
+  return parseJSON(
+    filter(split(decodeText(it), '\n'), (line) => Boolean(line && line.trim())),
+    options,
+  )
 }

packages/@sanity/migrate/src/runner/dryRun.ts

@@ -2,7 +2,7 @@ import {CompactFormatter} from '@bjoerge/mutiny'
 import {SanityDocument} from '@sanity/types'
 import {APIConfig, Migration} from '../types'
 import {ndjson} from '../it-utils/ndjson'
-import {fromExportEndpoint} from '../sources/fromExportEndpoint'
+import {fromExportEndpoint, safeJsonParser} from '../sources/fromExportEndpoint'
 import {collectMigrationMutations} from './collectMigrationMutations'
 
 interface MigrationRunnerOptions {
@@ -12,7 +12,9 @@ interface MigrationRunnerOptions {
 export async function* dryRun(config: MigrationRunnerOptions, migration: Migration) {
   const mutations = collectMigrationMutations(
     migration,
-    ndjson(await fromExportEndpoint(config.api)) as AsyncIterableIterator<SanityDocument>,
+    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {
+      parse: safeJsonParser,
+    }),
   )
 
   for await (const mutation of mutations) {

packages/@sanity/migrate/src/runner/run.ts

@@ -2,7 +2,7 @@ import {SanityDocument} from '@sanity/types'
 import {MultipleMutationResult} from '@sanity/client'
 import {APIConfig, Migration} from '../types'
 import {ndjson} from '../it-utils/ndjson'
-import {fromExportEndpoint} from '../sources/fromExportEndpoint'
+import {fromExportEndpoint, safeJsonParser} from '../sources/fromExportEndpoint'
 import {toMutationEndpoint} from '../destinations/toMutationEndpoint'
 import {collectMigrationMutations} from './collectMigrationMutations'
 
@@ -13,7 +13,9 @@ interface MigrationRunnerOptions {
 export async function* run(config: MigrationRunnerOptions, migration: Migration) {
   const mutations = collectMigrationMutations(
     migration,
-    ndjson(await fromExportEndpoint(config.api)) as AsyncIterableIterator<SanityDocument>,
+    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {
+      parse: safeJsonParser,
+    }),
   )
 
   for await (const result of toMutationEndpoint(config.api, mutations)) {

packages/@sanity/migrate/src/sources/fromExportEndpoint.ts

@@ -1,7 +1,9 @@
+import {createSafeJsonParser} from '@sanity/util/createSafeJsonParser'
 import {fetchAsyncIterator} from '../fetch-utils/fetchStream'
 import {toFetchOptions} from '../fetch-utils/sanityRequestOptions'
 import {endpoints} from '../fetch-utils/endpoints'
 import {APIConfig} from '../types'
+import {SanityDocument} from '@sanity/types'
 
 export function fromExportEndpoint(options: APIConfig) {
   return fetchAsyncIterator(
@@ -14,3 +16,15 @@ export function fromExportEndpoint(options: APIConfig) {
     }),
   )
 }
+
+/**
+ * Safe JSON parser that is able to handle lines interrupted by an error object.
+ *
+ * This may occur when streaming NDJSON from the Export HTTP API.
+ *
+ * @internal
+ * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
+ */
+export const safeJsonParser = createSafeJsonParser<SanityDocument>({
+  errorLabel: 'Error streaming dataset',
+})

packages/@sanity/util/.gitignore

@@ -13,6 +13,7 @@
 
 # Exports
 /content.js
+/createSafeJsonParser.js
 /fs.js
 /legacyDateFormat.js
 /paths.js

packages/@sanity/util/exports/createSafeJsonParser.ts

@@ -0,0 +1 @@
+export * from '../src/createSafeJsonParser'

packages/@sanity/util/package.json

@@ -2,14 +2,7 @@
   "name": "@sanity/util",
   "version": "3.26.1",
   "description": "Utilities shared across projects of Sanity",
-  "keywords": [
-    "sanity",
-    "cms",
-    "headless",
-    "realtime",
-    "content",
-    "util"
-  ],
+  "keywords": ["sanity", "cms", "headless", "realtime", "content", "util"],
   "homepage": "https://www.sanity.io/",
   "bugs": {
     "url": "https://github.com/sanity-io/sanity/issues"
@@ -55,6 +48,17 @@
       "import": "./lib/content.esm.js",
       "default": "./lib/content.esm.js"
     },
+    "./createSafeJsonParser": {
+      "types": "./lib/exports/createSafeJsonParser.d.ts",
+      "source": "./exports/createSafeJsonParser.ts",
+      "require": "./lib/createSafeJsonParser.js",
+      "node": {
+        "module": "./lib/createSafeJsonParser.esm.js",
+        "import": "./lib/createSafeJsonParser.cjs.mjs"
+      },
+      "import": "./lib/createSafeJsonParser.esm.js",
+      "default": "./lib/createSafeJsonParser.esm.js"
+    },
     "./legacyDateFormat": {
       "types": "./lib/exports/legacyDateFormat.d.ts",
       "source": "./exports/legacyDateFormat.ts",
@@ -85,22 +89,16 @@
   "types": "./lib/exports/index.d.ts",
   "typesVersions": {
     "*": {
-      "fs": [
-        "./lib/exports/fs.d.ts"
-      ],
-      "content": [
-        "./lib/exports/content.d.ts"
-      ],
-      "legacyDateFormat": [
-        "./lib/exports/legacyDateFormat.d.ts"
-      ],
-      "paths": [
-        "./lib/exports/paths.d.ts"
-      ]
+      "fs": ["./lib/exports/fs.d.ts"],
+      "content": ["./lib/exports/content.d.ts"],
+      "createSafeJsonParser": ["./lib/exports/createSafeJsonParser.d.ts"],
+      "legacyDateFormat": ["./lib/exports/legacyDateFormat.d.ts"],
+      "paths": ["./lib/exports/paths.d.ts"]
     }
   },
   "files": [
     "content.js",
+    "createSafeJsonParser.js",
     "fs.js",
     "legacyDateFormat.js",
     "lib",
@@ -112,7 +110,7 @@
     "build": "pkg-utils build --tsconfig tsconfig.lib.json",
     "postbuild": "run-s check:package",
     "check:package": "pkg-utils --tsconfig tsconfig.lib.json",
-    "clean": "rimraf content.js fs.js legacyDateFormat.js lib paths.js",
+    "clean": "rimraf content.js createSafeJsonParser.js fs.js legacyDateFormat.js lib paths.js",
     "test": "jest",
     "watch": "pkg-utils watch --tsconfig tsconfig.lib.json"
   },

packages/@sanity/migrate/src/it-utils/createSafeJsonParser.ts → packages/@sanity/util/src/createSafeJsonParser.ts

@@ -7,8 +7,10 @@ type Parser<Type> = (line: string) => Type
 /**
  * Create a safe JSON parser that is able to handle lines interrupted by an error object.
  *
- * TODO: Unify with the `tryParseJson` function that already exists at
- * `packages/@sanity/export/src/tryParseJson.js`.
+ * This may occur when streaming NDJSON from the Export HTTP API.
+ *
+ * @internal
+ * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request}
  */
 export function createSafeJsonParser<Type>({errorLabel}: Options): Parser<Type> {
   return function safeJsonParser(line) {

packages/@sanity/util/test/createSafeJsonParser.test.ts

@@ -0,0 +1,20 @@
+import {createSafeJsonParser} from '../src/createSafeJsonParser'
+
+const parse = createSafeJsonParser({
+  errorLabel: 'Error parsing JSON',
+})
+
+test('parse JSON', () => {
+  expect(parse('{"someString": "string"}')).toEqual({someString: 'string'})
+  expect(parse('{"someNumber": 42}')).toEqual({someNumber: 42})
+})
+
+test('parse JSON with interrupting error', () => {
+  expect(() => parse('{"someString": "str{"error":{"description":"Some error"}}'))
+    .toThrowErrorMatchingInlineSnapshot(`
+"Error parsing JSON: Some error
+
+{\\"error\\":{\\"description\\":\\"Some error\\"}}
+"
+`)
+})

packages/@sanity/util/turbo.json

@@ -3,7 +3,15 @@
   "extends": ["//"],
   "pipeline": {
     "build": {
-      "outputs": ["lib/**", "index.js", "content.js", "fs.js", "legacyDateFormat.js", "paths.js"]
+      "outputs": [
+        "lib/**",
+        "index.js",
+        "content.js",
+        "createSafeJsonParser.js",
+        "fs.js",
+        "legacyDateFormat.js",
+        "paths.js"
+      ]
     }
   }
 }

packages/sanity/src/_internal/cli/commands/migration/runMigrationCommand.ts

@@ -6,6 +6,7 @@ import {
   collectMigrationMutations,
   fromExportArchive,
   fromExportEndpoint,
+  safeJsonParser,
   Migration,
   ndjson,
   run,
@@ -164,7 +165,9 @@ async function* dryRun(
 ) {
   const mutations = collectMigrationMutations(
     migration,
-    ndjson(await fromExportEndpoint(config.api)) as AsyncIterableIterator<SanityDocument>,
+    ndjson<SanityDocument>(await fromExportEndpoint(config.api), {
+      parse: safeJsonParser,
+    }),
   )
 
   for await (const mutation of mutations) {