From 7ca5bbf0e0f5c24a0bda5de9142703a913ce3fa8 Mon Sep 17 00:00:00 2001
From: Ben Christel <benchristel@users.noreply.github.com>
Date: Tue, 19 Nov 2024 08:39:32 -0800
Subject: [PATCH] ADR 773: Look for the shortest assessment item to repro each
 parse error (#1872)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The exhaustive test tool for PerseusItem parsing outputs the `assessmentItem`s
that can't be parsed. The plan is to generate regression tests from these
assessment items.

Previously, the tool would output an arbitrary `assessmentItem` for each
failure.  This PR makes it keep the *shortest* `assessmentItem` instead. That
way, our regression test data won't be over-complicated.

Issue: none

## Test plan:

Run the exhaustive test tool on a subset of data.

Author: benchristel

Reviewers: benchristel, jeremywiebe, anakaren-rojas, catandthemachines, nishasy

Required Reviewers:

Approved By: jeremywiebe

Checks: ✅ Publish npm snapshot (ubuntu-latest, 20.x), ✅ Lint, Typecheck, Format, and Test (ubuntu-latest, 20.x), ✅ Cypress (ubuntu-latest, 20.x), ✅ Check for .changeset entries for all changed files (ubuntu-latest, 20.x), ✅ Check builds for changes in size (ubuntu-latest, 20.x), ✅ Publish Storybook to Chromatic (ubuntu-latest, 20.x), ✅ gerald

Pull Request URL: https://github.com/Khan/perseus/pull/1872
---
 .changeset/late-sheep-rush.md                 |  5 +++++
 .../exhaustive-test-tool/index.ts             | 21 ++++++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/late-sheep-rush.md

diff --git a/.changeset/late-sheep-rush.md b/.changeset/late-sheep-rush.md
new file mode 100644
index 0000000000..07c7162a8b
--- /dev/null
+++ b/.changeset/late-sheep-rush.md
@@ -0,0 +1,5 @@
+---
+"@khanacademy/perseus": patch
+---
+
+Internal: make the exhaustive test tool for PerseusItem parsing find the shortest input file that repros each failure
diff --git a/packages/perseus/src/util/parse-perseus-json/exhaustive-test-tool/index.ts b/packages/perseus/src/util/parse-perseus-json/exhaustive-test-tool/index.ts
index e3d5880d09..f50ea2bede 100755
--- a/packages/perseus/src/util/parse-perseus-json/exhaustive-test-tool/index.ts
+++ b/packages/perseus/src/util/parse-perseus-json/exhaustive-test-tool/index.ts
@@ -75,7 +75,11 @@ async function testFile(path: string, outputDir: string) {
                 desc,
                 "utf-8",
             );
-            await fs.writeFile(
+            // We'd like to find the shortest assessment item that reproduces
+            // each mismatch, to keep our regression tests succinct and
+            // readable. To that end, we only update the item.json file if the
+            // current item is shorter than the one already on disk.
+            await writeFileIfShorterThanExisting(
                 join(outputDir, hash, "item.json"),
                 String(JSON.stringify(rawItem)),
                 "utf-8",
@@ -124,6 +128,21 @@ function typeName(value: unknown): string {
     return typeof value;
 }
 
+async function writeFileIfShorterThanExisting(
+    path: string,
+    content: string,
+    encoding: "utf-8", // TODO(benchristel): add more encodings if needed
+) {
+    const size = await fs
+        .stat(path)
+        .then((stat) => stat.size)
+        .catch(() => null);
+    const contentLength = Buffer.byteLength(content, encoding);
+    if (size == null || contentLength < size) {
+        await fs.writeFile(path, content, encoding);
+    }
+}
+
 async function* listFilesRecursively(dir: string) {
     for await (const entry of await fs.opendir(dir, {recursive: true})) {
         if (entry.isFile()) {