feat: return invalid document in error message (#248)

https://coveord.atlassian.net/browse/CDX-1284

src/__stub__/jsondocuments/brokenBatch.json

@@ -0,0 +1,14 @@
+[
+  {
+    "DocumentId": "https://www.example.com/foo",
+    "title": "Foo"
+  },
+  {
+    "DocumentId": "https://www.example.com/bar",
+    "Broken_title": "Bar"
+  },
+  {
+    "DocumentId": "https://www.example.com/baz",
+    "title": "Baz"
+  }
+]

src/errors/validatorErrors.ts

@@ -1,3 +1,4 @@
+import {PrimitivesValues} from '@coveo/bueno';
 import {PathLike} from 'fs';
 import {PushApiClientBaseError} from './baseError';
 
@@ -16,10 +17,17 @@ export class NotAJsonFileError extends PushApiClientBaseError {
 
 export class InvalidDocument extends PushApiClientBaseError {
   public name = 'Invalid JSON Document Error';
-  public constructor(p: PathLike, explanation: string) {
+  public constructor(
+    p: PathLike,
+    doc: Record<string, PrimitivesValues>,
+    explanation: string
+  ) {
     super(
       [
         `${p} is not a valid JSON: ${explanation}`,
+        'Document in error:',
+        JSON.stringify(doc, null, 2),
+        '',
         'Helpful links on the expected JSON format:',
         ' • JSON file example: https://github.com/coveo/push-api-client.js/tree/main/samples/json',
         ' • Document Body reference: https://docs.coveo.com/en/75#documentbody',

src/validation/__snapshots__/parseFile.spec.ts.snap

@@ -1,5 +1,18 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`parseFile should fail on reserved keyword 1`] = `
+brokenBatch.json is not a valid JSON: Document contains an invalid value for title: value is required.
+Document in error:
+{
+  "DocumentId": "https://www.example.com/bar",
+  "Broken_title": "Bar"
+}
+
+Helpful links on the expected JSON format:
+ • JSON file example: https://github.com/coveo/push-api-client.js/tree/main/samples/json
+ • Document Body reference: https://docs.coveo.com/en/75#documentbody
+`;
+
 exports[`parseFile should fail on unsupported metadata key 1`] = `
 "
 The following field names are still invalid after transformation:

src/validation/caseInsensitiveDocument.ts

@@ -3,10 +3,36 @@ export type Recordable<T> = {
 };
 
 export class CaseInsensitiveDocument<T> {
-  public documentRecord: Record<string, T> = {};
+  private _originalDocument: Record<string, T> = {};
+  private _remainingRecord: Record<string, T> = {};
+
   public constructor(doc: Record<string, T>) {
+    this._originalDocument = this.shallowCopy(doc);
+
     Object.entries(doc).forEach(([k, v]) => {
-      this.documentRecord[k.toLowerCase()] = v;
+      this._remainingRecord[k.toLowerCase()] = v;
     });
   }
+
+  public remove(...keys: string[]) {
+    keys.forEach((key) => {
+      delete this._remainingRecord[key.toLowerCase()];
+    });
+  }
+
+  public getRecordValue(key: string) {
+    return this._remainingRecord[key.toLowerCase()];
+  }
+
+  public get remainingRecord(): Readonly<Record<string, T>> {
+    return this._remainingRecord;
+  }
+
+  public get originalDocument(): Readonly<Record<string, T>> {
+    return this._originalDocument;
+  }
+
+  private shallowCopy(sourceRecord: Record<string, T>) {
+    return Object.assign({}, sourceRecord);
+  }
 }

src/validation/knownKey.ts

@@ -16,10 +16,10 @@ export class KnownKeys<T extends PrimitivesValues> {
 
   public get value() {
     const found = this.keys.find(
-      (k) => !isNullOrUndefined(this.doc.documentRecord[k.toLowerCase()])
+      (k) => !isNullOrUndefined(this.doc.getRecordValue(k))
     );
     if (found) {
-      return this.doc.documentRecord[found.toLowerCase()];
+      return this.doc.getRecordValue(found);
     }
 
     return null;
@@ -36,7 +36,7 @@ export class KnownKeys<T extends PrimitivesValues> {
   public whenDoesNotExist<U = T>(cb: (v: U) => void) {
     const value = this.value;
     if (isNullOrUndefined(value)) {
-      cb(this.doc.documentRecord as U);
+      cb(this.doc.remainingRecord as U);
     }
     return this;
   }

src/validation/parseFile.spec.ts

@@ -2,6 +2,17 @@ import {join} from 'path';
 import {cwd} from 'process';
 import {parseAndGetDocumentBuilderFromJSONDocument} from './parseFile';
 import {InvalidDocument} from '../errors/validatorErrors';
+import {readFileSync} from 'fs';
+
+const stripFilePathFromSnapshot = () => {
+  const fileJsonFileErrorRegex = new RegExp(
+    /^.*?(\w+\.json)(?= is not a valid JSON)/m
+  );
+  expect.addSnapshotSerializer({
+    test: (val: string) => Boolean(val.match(fileJsonFileErrorRegex)),
+    print: (val) => `${val}`.replace(fileJsonFileErrorRegex, '$1'),
+  });
+};
 
 describe('parseFile', () => {
   const pathToStub = join(cwd(), 'src', '__stub__');
@@ -69,9 +80,11 @@ describe('parseFile', () => {
     },
   ])('$title', async ({fileName, error}) => {
     const file = join(pathToStub, 'jsondocuments', fileName);
+    const fileContent = readFileSync(file).toString();
     await expect(parse(file)).rejects.toThrow(
       new InvalidDocument(
         file,
+        JSON.parse(fileContent),
         `Document contains an invalid value for ${error}`
       )
     );
@@ -86,14 +99,22 @@ describe('parseFile', () => {
 
   it('should fail on reserved keyword', async () => {
     const file = join(pathToStub, 'jsondocuments', 'reservedKeyword.json');
+    const fileContent = readFileSync(file).toString();
     await expect(parse(file)).rejects.toThrow(
       new InvalidDocument(
         file,
+        JSON.parse(fileContent),
         'Cannot use parentid as a metadata key: It is a reserved key name. See https://docs.coveo.com/en/78/index-content/push-api-reference#json-document-reserved-key-names'
       )
     );
   });
 
+  it('should fail on reserved keyword', async () => {
+    const file = join(pathToStub, 'jsondocuments', 'brokenBatch.json');
+    stripFilePathFromSnapshot();
+    await expect(parse(file)).rejects.toThrowErrorMatchingSnapshot();
+  });
+
   it('should fail on unsupported metadata key', async () => {
     const file = join(pathToStub, 'jsondocuments', 'invalidFields.json');
     await expect(parse(file)).rejects.toThrowErrorMatchingSnapshot();

src/validation/parseFile.ts

@@ -70,7 +70,11 @@ const processDocument = (
     processMetadata(caseInsensitiveDoc, documentBuilder, options);
   } catch (error) {
     if (typeof error === 'string') {
-      throw new InvalidDocument(documentPath, error);
+      throw new InvalidDocument(
+        documentPath,
+        caseInsensitiveDoc.originalDocument,
+        error
+      );
     }
     throw error;
   }
@@ -88,7 +92,11 @@ const validateRequiredKeysAndGetDocumentBuilder = (
   );
 
   if (!requiredDocumentId.isValid) {
-    throw new InvalidDocument(documentPath, requiredDocumentId.explanation);
+    throw new InvalidDocument(
+      documentPath,
+      caseInsensitiveDoc.originalDocument,
+      requiredDocumentId.explanation
+    );
   }
 
   const requiredDocumentTitle = new RequiredKeyValidator<string>(
@@ -97,12 +105,14 @@ const validateRequiredKeysAndGetDocumentBuilder = (
     new StringValue({required: true, emptyAllowed: false})
   );
   if (!requiredDocumentTitle.isValid) {
-    throw new InvalidDocument(documentPath, requiredDocumentTitle.explanation);
+    throw new InvalidDocument(
+      documentPath,
+      caseInsensitiveDoc.originalDocument,
+      requiredDocumentTitle.explanation
+    );
   }
 
-  delete caseInsensitiveDoc.documentRecord['documentid'];
-  delete caseInsensitiveDoc.documentRecord['uri'];
-  delete caseInsensitiveDoc.documentRecord['title'];
+  caseInsensitiveDoc.remove('documentid', 'uri', 'title');
 
   return new DocumentBuilder(
     requiredDocumentId.value!,
@@ -116,38 +126,38 @@ const processKnownKeys = (
 ) => {
   new KnownKeys<string>('author', caseInsensitiveDoc).whenExists((author) => {
     documentBuilder.withAuthor(author);
-    delete caseInsensitiveDoc.documentRecord['author'];
+    caseInsensitiveDoc.remove('author');
   });
   new KnownKeys<string>('clickableuri', caseInsensitiveDoc).whenExists(
     (clickuri) => {
       documentBuilder.withClickableUri(clickuri);
-      delete caseInsensitiveDoc.documentRecord['clickableuri'];
+      caseInsensitiveDoc.remove('clickableuri');
     }
   );
   new KnownKeys<string>('data', caseInsensitiveDoc).whenExists((data) => {
     documentBuilder.withData(data);
-    delete caseInsensitiveDoc.documentRecord['data'];
+    caseInsensitiveDoc.remove('data');
   });
   new KnownKeys<string>('date', caseInsensitiveDoc).whenExists((date) => {
     documentBuilder.withDate(date);
-    delete caseInsensitiveDoc.documentRecord['date'];
+    caseInsensitiveDoc.remove('date');
   });
   new KnownKeys<string>('modifieddate', caseInsensitiveDoc).whenExists(
     (modifiedDate) => {
       documentBuilder.withModifiedDate(modifiedDate);
-      delete caseInsensitiveDoc.documentRecord['modifieddate'];
+      caseInsensitiveDoc.remove('modifieddate');
     }
   );
   new KnownKeys<string>('fileextension', caseInsensitiveDoc).whenExists(
     (fileExtension) => {
       documentBuilder.withFileExtension(fileExtension);
-      delete caseInsensitiveDoc.documentRecord['fileextension'];
+      caseInsensitiveDoc.remove('fileextension');
     }
   );
   new KnownKeys<string>('permanentid', caseInsensitiveDoc).whenExists(
     (permanentId) => {
       documentBuilder.withPermanentId(permanentId);
-      delete caseInsensitiveDoc.documentRecord['permanentid'];
+      caseInsensitiveDoc.remove('permanentid');
     }
   );
 };
@@ -158,7 +168,7 @@ const processMetadata = (
   options?: ParseDocumentOptions
 ) => {
   const metadata: Metadata = {};
-  Object.entries(caseInsensitiveDoc.documentRecord).forEach(([k, v]) => {
+  Object.entries(caseInsensitiveDoc.remainingRecord).forEach(([k, v]) => {
     metadata[k] = v! as Extract<PrimitivesValues, MetadataValue>;
   });
   documentBuilder.withMetadata(metadata, options?.fieldNameTransformer);