feat(cli): new documents validate command (#5475)

* feat(cli): new documents validate command

* feat(cli): finish implementing pretty reporter

* fix(cli): improve mock browser env support

* feat(cli): add --level flag

* fix(cli): throw if workspace name mismatches

* feat(cli): conditionally add validators based on environment

* feat(cli): add concurrencly limiter to custom validators

* feat(cli): add validation json reporter

* fix(cli): fix bug with summary text

* refactor(cli): add concurrency limiter to batched getDocumentExists

* refactor(cli): fetch reference existence upfront; stream export to file

* test(cli): update validateDocument tests

* docs(cli): update workerChannel ts docs

* feat(cli): finish CLI help text

* fix(cli): update to type-only imports

* fix(cli): add environment to validate call

* feat(cli): add prompt

* fix(cli): use log-symbols

packages/@sanity/types/src/validation/types.ts

@@ -152,7 +152,7 @@ export interface Rule {
   either(children: Rule[]): Rule
   optional(): Rule
   required(): Rule
-  custom<T = unknown>(fn: CustomValidator<T>): Rule
+  custom<T = unknown>(fn: CustomValidator<T>, options?: {bypassConcurrencyLimit?: boolean}): Rule
   min(len: number | FieldReference): Rule
   max(len: number | FieldReference): Rule
   length(len: number | FieldReference): Rule
@@ -175,7 +175,21 @@ export interface Rule {
   reference(): Rule
   fields(rules: FieldRules): Rule
   assetRequired(): Rule
-  validate(value: unknown, options: ValidationContext): Promise<ValidationMarker[]>
+  validate(
+    value: unknown,
+    options: ValidationContext & {
+      /**
+       * @deprecated Internal use only
+       * @internal
+       */
+      __internal?: {
+        customValidationConcurrencyLimiter?: {
+          ready: () => Promise<void>
+          release: () => void
+        }
+      }
+    },
+  ): Promise<ValidationMarker[]>
 }
 
 /** @public */
@@ -254,6 +268,7 @@ export interface ValidationContext {
   document?: SanityDocument
   path?: Path
   getDocumentExists?: (options: {id: string}) => Promise<boolean>
+  environment: 'cli' | 'studio'
 }
 
 /**
@@ -381,10 +396,10 @@ export type CustomValidatorResult =
   | LocalizedValidationMessages
 
 /** @public */
-export type CustomValidator<T = unknown> = (
-  value: T,
-  context: ValidationContext,
-) => CustomValidatorResult | Promise<CustomValidatorResult>
+export interface CustomValidator<T = unknown> {
+  (value: T, context: ValidationContext): CustomValidatorResult | Promise<CustomValidatorResult>
+  bypassConcurrencyLimit?: boolean
+}
 
 /** @public */
 export interface SlugValidationContext extends ValidationContext {

packages/sanity/package.config.ts

@@ -28,6 +28,11 @@ export default defineConfig({
       require: './lib/_internal/cli/threads/getGraphQLAPIs.js',
       default: './lib/_internal/cli/threads/getGraphQLAPIs.js',
     },
+    './_internal/cli/threads/validateDocuments': {
+      source: './src/_internal/cli/threads/validateDocuments.ts',
+      require: './lib/_internal/cli/threads/validateDocuments.js',
+      default: './lib/_internal/cli/threads/validateDocuments.js',
+    },
   }),
 
   extract: {

packages/sanity/package.json

@@ -249,6 +249,7 @@
     "observable-callback": "^1.0.1",
     "oneline": "^1.0.3",
     "open": "^8.4.0",
+    "p-map": "^7.0.0",
     "pirates": "^4.0.0",
     "pluralize-esm": "^9.0.2",
     "polished": "^4.2.2",

packages/sanity/src/_internal/cli/actions/validation/reporters/index.ts

@@ -0,0 +1,5 @@
+import {pretty} from './prettyReporter'
+import {ndjson} from './ndjsonReporter'
+import {json} from './jsonReporter'
+
+export const reporters = {pretty, ndjson, json}

packages/sanity/src/_internal/cli/actions/validation/reporters/jsonReporter.ts

@@ -0,0 +1,39 @@
+import type {ValidationMarker} from '@sanity/types'
+import type {BuiltInValidationReporter} from '../validateAction'
+
+export const json: BuiltInValidationReporter = async ({output, worker}) => {
+  let overallLevel: 'error' | 'warning' | 'info' = 'info'
+
+  const results: Array<{
+    documentId: string
+    documentType: string
+    revision: string
+    level: 'error' | 'warning' | 'info'
+    markers: ValidationMarker[]
+  }> = []
+
+  for await (const {
+    documentId,
+    documentType,
+    markers,
+    revision,
+    level,
+  } of worker.stream.validation()) {
+    if (level === 'error') overallLevel = 'error'
+    if (level === 'warning' && overallLevel !== 'error') overallLevel = 'warning'
+
+    results.push({
+      documentId,
+      documentType,
+      revision,
+      level,
+      markers,
+    })
+  }
+
+  await worker.dispose()
+
+  output.print(JSON.stringify(results))
+
+  return overallLevel
+}

packages/sanity/src/_internal/cli/actions/validation/reporters/ndjsonReporter.ts

@@ -0,0 +1,24 @@
+import type {BuiltInValidationReporter} from '../validateAction'
+
+export const ndjson: BuiltInValidationReporter = async ({output, worker}) => {
+  let overallLevel: 'error' | 'warning' | 'info' = 'info'
+
+  for await (const {
+    documentId,
+    documentType,
+    markers,
+    revision,
+    level,
+  } of worker.stream.validation()) {
+    if (level === 'error') overallLevel = 'error'
+    if (level === 'warning' && overallLevel !== 'error') overallLevel = 'warning'
+
+    if (markers.length) {
+      output.print(JSON.stringify({documentId, documentType, revision, level, markers}))
+    }
+  }
+
+  await worker.dispose()
+
+  return overallLevel
+}

packages/sanity/src/_internal/cli/actions/validation/reporters/prettyReporter/__tests__/formatDocumentValidation.test.ts

@@ -0,0 +1,72 @@
+import {formatDocumentValidation} from '../formatDocumentValidation'
+
+// disables some terminal specific things that are typically auto detected
+jest.mock('tty', () => ({isatty: () => false}))
+jest.mock('chalk', () => {
+  const chalk = jest.requireActual('chalk')
+  chalk.level = 0
+  return chalk
+})
+
+describe('formatDocumentValidation', () => {
+  it('formats a set of markers in to a printed tree, sorting markers, and adding spacing', () => {
+    const result = formatDocumentValidation({
+      basePath: '/test',
+      documentId: 'my-document-id',
+      documentType: 'person',
+      level: 'error',
+      revision: 'rev',
+      studioHost: null,
+      markers: [
+        {level: 'error', message: 'Top-level marker', path: []},
+        {level: 'error', message: '2nd top-level marker', path: []},
+        {level: 'error', message: 'Property marker', path: ['foo']},
+        {level: 'error', message: 'Nested marker', path: ['bar', 'title']},
+        {level: 'error', message: '2nd nested marker', path: ['bar', 'title']},
+        {level: 'error', message: '2nd property marker', path: ['baz']},
+        {level: 'warning', message: 'Warning', path: ['beep', 'boop']},
+        {level: 'error', message: 'Errors sorted first', path: ['beep', 'boop']},
+      ],
+    })
+
+    expect(result).toEqual(
+      `
+[ERROR] [person] my-document-id
+│  (root) ........................ ✖ Top-level marker
+│                                  ✖ 2nd top-level marker
+├─ foo ........................... ✖ Property marker
+├─ bar
+│ └─ title ....................... ✖ Nested marker
+│                                  ✖ 2nd nested marker
+├─ baz ........................... ✖ 2nd property marker
+└─ beep
+  └─ boop ........................ ✖ Errors sorted first
+                                   ⚠ Warning`.trim(),
+    )
+  })
+
+  it('formats a set of top-level markers only (should have an elbow at first message)', () => {
+    const result = formatDocumentValidation({
+      basePath: '/test',
+      documentId: 'my-document-id',
+      documentType: 'person',
+      level: 'error',
+      revision: 'rev',
+      studioHost: null,
+      markers: [
+        {level: 'info', message: '2nd top-level marker (should come last)', path: []},
+        {level: 'error', message: 'Lone top-level marker (should get elbow)', path: []},
+        {level: 'warning', message: 'Warning, should come second', path: []},
+      ],
+    })
+
+    expect(result).toEqual(
+      `
+[ERROR] [person] my-document-id
+└─ (root) ........................ ✖ Lone top-level marker (should get elbow)
+                                   ⚠ Warning, should come second
+                                   ℹ 2nd top-level marker (should come last)
+`.trim(),
+    )
+  })
+})

packages/sanity/src/_internal/cli/actions/validation/reporters/prettyReporter/formatDocumentValidation.ts

@@ -0,0 +1,176 @@
+import chalk from 'chalk'
+import {ValidationMarker} from '@sanity/types'
+import logSymbols from 'log-symbols'
+import {DocumentValidationResult, Level, isTty, levelValues} from './util'
+import {pathToString} from 'sanity'
+
+export interface FormatDocumentValidationOptions extends DocumentValidationResult {
+  studioHost: string | null
+  basePath: string
+}
+
+interface ValidationTree {
+  markers?: Pick<ValidationMarker, 'level' | 'message'>[]
+  children?: Record<string, ValidationTree>
+}
+
+const levelHeaders = {
+  error: isTty ? chalk.bold(chalk.bgRed(' ERROR ')) : chalk.red('[ERROR]'),
+  warning: isTty ? chalk.bold(chalk.bgYellow(' WARN ')) : chalk.yellow('[WARN]'),
+  info: isTty ? chalk.bold(chalk.cyan(' INFO ')) : chalk.cyan('[INFO]'),
+}
+/**
+ * Creates a terminal hyperlink. Only outputs a hyperlink if the output is
+ * determined to be a TTY
+ */
+const link = (text: string, url: string) =>
+  isTty ? `\u001b]8;;${url}\u0007${text}\u001b]8;;\u0007` : chalk.underline(text)
+
+/**
+ * Recursively calculates the max length of all the keys in the given validation
+ * tree respecting extra length due to indentation depth. Used to calculate the
+ * padding for the rest of the tree.
+ */
+const maxKeyLength = (children: Record<string, ValidationTree> = {}, depth = 0): number => {
+  return Object.entries(children)
+    .map(([key, child]) =>
+      Math.max(key.length + depth * 2, maxKeyLength(child.children, depth + 1)),
+    )
+    .reduce((max, next) => (next > max ? next : max), 0)
+}
+
+/**
+ * For sorting markers
+ */
+const compareLevels = <T extends {level: Level; message: string}>(a: T, b: T) =>
+  levelValues[a.level] - levelValues[b.level]
+
+/**
+ * Recursively formats a given tree into a printed user-friendly tree structure
+ */
+const formatTree = (
+  node: Record<string, ValidationTree> = {},
+  paddingLength: number,
+  indent = '',
+): string => {
+  const entries = Object.entries(node)
+
+  return entries
+    .map(([key, child], index) => {
+      const isLast = index === entries.length - 1
+      const nextIndent = `${indent}${isLast ? '  ' : '│ '}`
+      const nested = formatTree(child.children, paddingLength, nextIndent)
+
+      if (!child.markers?.length) {
+        const current = `${indent}${isLast ? '└' : '├'}─ ${key}`
+        return [current, nested].filter(Boolean).join('\n')
+      }
+
+      const [first, ...rest] = child.markers.slice().sort(compareLevels)
+      const firstPadding = '.'.repeat(paddingLength - indent.length - key.length)
+      const elbow = isLast ? '└' : '├'
+      const firstBullet = logSymbols[first.level]
+      const subsequentPadding = ' '.repeat(paddingLength - indent.length + 2)
+
+      const firstMessage = `${indent}${elbow}─ ${key} ${firstPadding} ${firstBullet} ${first.message}`
+      const subsequentMessages = rest
+        .map(
+          (marker) =>
+            `${nextIndent}${subsequentPadding} ${logSymbols[marker.level]} ${marker.message}`,
+        )
+        .join('\n')
+
+      const current = [firstMessage, subsequentMessages].filter(Boolean).join('\n')
+      return [current, nested].filter(Boolean).join('\n')
+    })
+    .join('\n')
+}
+
+/**
+ * Formats the markers at the root of the validation tree
+ */
+const formatRootErrors = (root: ValidationTree, hasChildren: boolean, paddingLength: number) => {
+  if (!root.markers) return ''
+
+  const [first, ...rest] = root.markers.slice().sort(compareLevels)
+  if (!first) return ''
+
+  const firstElbow = hasChildren ? '│ ' : '└─'
+  const firstPadding = '.'.repeat(paddingLength - 6)
+  const firstLine = `${firstElbow} (root) ${firstPadding} ${logSymbols[first.level]} ${
+    first.message
+  }`
+  const subsequentPadding = ' '.repeat(paddingLength + 2)
+  const subsequentElbow = hasChildren ? '│ ' : '  '
+
+  const restOfLines = rest
+    .map(
+      (marker) =>
+        `${subsequentElbow}${subsequentPadding} ${logSymbols[marker.level]} ${marker.message}`,
+    )
+    .join('\n')
+  return [firstLine, restOfLines].filter(Boolean).join('\n')
+}
+
+/**
+ * Converts a set of markers with paths into a tree of markers where the paths
+ * are embedded in the tree
+ */
+function convertToTree(markers: ValidationMarker[]): ValidationTree {
+  const root: ValidationTree = {}
+
+  // add the markers to the tree
+  function addMarker(marker: ValidationMarker, node: ValidationTree = root) {
+    // if we've traversed the whole path
+    if (!marker.path.length) {
+      if (!node.markers) node.markers = [] // ensure markers is defined
+
+      // then add the marker to the front
+      node.markers.push({level: marker.level, message: marker.message})
+      return
+    }
+
+    const [current, ...rest] = marker.path
+    const key = pathToString([current])
+
+    // ensure the current node has children and the next node
+    if (!node.children) node.children = {}
+    if (!(key in node.children)) node.children[key] = {}
+
+    addMarker({...marker, path: rest}, node.children[key])
+  }
+
+  for (const marker of markers) addMarker(marker)
+  return root
+}
+
+/**
+ * Formats document validation results into a user-friendly tree structure
+ */
+export function formatDocumentValidation({
+  basePath,
+  documentId,
+  documentType,
+  level,
+  studioHost,
+  markers,
+}: FormatDocumentValidationOptions): string {
+  const tree = convertToTree(markers)
+  const editLink =
+    studioHost &&
+    `${studioHost}${basePath}/intent/edit/id=${encodeURIComponent(
+      documentId,
+    )};type=${encodeURIComponent(documentType)}`
+
+  const documentTypeHeader = isTty ? chalk.bgWhite(` ${documentType} `) : `[${documentType}]`
+
+  const header = `${levelHeaders[level]} ${documentTypeHeader} ${
+    editLink ? link(documentId, editLink) : chalk.underline(documentId)
+  }`
+
+  const paddingLength = Math.max(maxKeyLength(tree.children) + 2, 30)
+  const childErrors = formatTree(tree.children, paddingLength)
+  const rootErrors = formatRootErrors(tree, childErrors.length > 0, paddingLength)
+
+  return [header, rootErrors, childErrors].filter(Boolean).join('\n')
+}

packages/sanity/src/_internal/cli/actions/validation/reporters/prettyReporter/index.ts

@@ -0,0 +1 @@
+export {pretty} from './prettyReporter'