fix: memory leak in large files with lots of errors (#180)

* fix: memory leak in large files with lots of errors

* fix: remove a mistanekly commited file

lib/validators/schema.js

@@ -6,6 +6,23 @@ const AjvDraft4 = require('ajv-draft-04');
 
 const { getSpecificationName } = require('../util');
 
+/**
+ * We've had issues with specs larger than 2MB+ with 1,000+ errors causing memory leaks so if we
+ * have a spec with more than `LARGE_SPEC_ERROR_CAP` errors and it's **stringified** length is
+ * larger than `LARGE_SPEC_LIMITS` then we will only return the first `LARGE_SPEC_ERROR_CAP` errors.
+ *
+ * Ideally we'd be looking at the byte size of the spec instead of looking at its stringified
+ * length value but the Blob API, which we'd use to get its size with `new Blob([str]).size;`, was
+ * only recently introduced in Node 15.
+ *
+ * w/r/t the 5,000,000 limit here: The spec we found causing these memory leaks had a size of
+ * 13,934,323 so 5mil seems like a decent cap to start with.
+ *
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/Blob}
+ */
+const LARGE_SPEC_ERROR_CAP = 20;
+const LARGE_SPEC_SIZE_CAP = 5000000;
+
 module.exports = validateSchema;
 
 /**
@@ -48,13 +65,33 @@ function validateSchema(api, options) {
   const isValid = ajv.validate(schema, api);
   if (!isValid) {
     const err = ajv.errors;
+
+    let additionalErrors = 0;
+    let reducedErrors = reduceAjvErrors(err);
+    if (reducedErrors.length >= LARGE_SPEC_ERROR_CAP) {
+      try {
+        if (JSON.stringify(api).length >= LARGE_SPEC_SIZE_CAP) {
+          additionalErrors = reducedErrors.length - 20;
+          reducedErrors = reducedErrors.slice(0, 20);
+        }
+      } catch (err) {
+        // If we failed to stringify the API definition to look at its size then we should process
+        // all of its errors as-is.
+      }
+    }
+
     let message = `${getSpecificationName(api)} schema validation failed.\n`;
     message += '\n';
-    message += betterAjvErrors(schema, api, reduceAjvErrors(err), {
+    message += betterAjvErrors(schema, api, reducedErrors, {
       colorize: options.validate.colorizeErrors,
       indent: 2,
     });
 
+    if (additionalErrors) {
+      message += '\n\n';
+      message += `Plus an additional ${additionalErrors} errors. Please resolve the above and re-run validation to see more.`;
+    }
+
     throw ono.syntax(err, { details: err }, message);
   }
 }

package.json

@@ -76,7 +76,7 @@
     "@apidevtools/openapi-schemas": "^2.1.0",
     "@apidevtools/swagger-methods": "^3.0.2",
     "@jsdevtools/ono": "^7.1.3",
-    "@readme/better-ajv-errors": "^1.5.0",
+    "@readme/better-ajv-errors": "^1.6.0",
     "@readme/json-schema-ref-parser": "^1.2.0",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",