From 138d026b55352453b21d25cdde3b21d6b4132a14 Mon Sep 17 00:00:00 2001 From: Howard Edwards Date: Thu, 30 Nov 2023 12:37:44 -0500 Subject: [PATCH] Support V2 Test Format build (#997) * Add scripts to convert a test plan from v1 to v2 format * Add and revise support and commands JSON data support.json: * add strings to support screen reader settings and assertion tokens. * Add URL data for ARIA and HTML-AAM specifications. commands.json: Add all the command string representations necessary for the new V2 test format. * WIP: Building test index page * Adding additional TODOs to be covered; start to addressing review page * Fix references after rebase * Add additional data validations * Use MUST/SHOULD/MAY on review page * Update validations * Now accounting for modeless ATs * Additional validations * Accounting for assertionExceptions * Additional validations and accounting for assertionExceptions; review page format now also matches 977 * Keys handling in test-io-format.mjs * Update support.json * Remove 'v2-made' files * Change 'NA' for modeless to 'defaultMode' * Format .mjs files * Fix lines in keys.mjs * Remove create-example-tests2 * Revert keys.mjs quotations * assertionTokens replacement * show replaced assertion tokens in reviewed tests * Conditionally show modeInstructions * testing * Set mode for commandInfo * Better support for modeless in aria-at-test-io-format.mjs and aria-at-test-run.mjs * Better support for modeless in test-reviewer.mjs * Formatting * Update v2maker to create a new row for commands previously defined as 'F / Shift + F' for eg. * Avoid setupScript thrown error when looking for script file * Formatting test-reviewer.mjs * Handle unexpected cases with v2 maker related scripts * Handle thrown error csv properties to pass validations (for now) * Additional cleanup * Fix pattern name identifier on index pagae * Formatting * Handle all 'at'-commands.csv files without looking specifically for known ats * Additional error handling for missing at-commands.csv files * Update jsdocs * Formatting * Conditionally build v2 over v1 test format * Refactor name for clarity * Edge case handling for test with multiple settings * Optimize edge case handling * Misc * Fix showing mode instructions on review page * Update validations * Update error message * Revert changes in package.json * Fix issues with Delete command validation * Remove unused function * Ensure rows in *-commands.csv are maintained by the presentationNumber * Fix case where titles and instructions were still unexpectedly referencing reading and interactions modes in v2maker script. This work has been duplicated in #990 * Fix review page to stop using generated commandPresentationNumbers * Update aria-at-test-run.mjs to drop support for generated command sequence numbers * Better support of 0-level assertion exceptions on review page * Fix linter * Remove 'modes' from v2 test file names * Remove comment * Add mode to individual *.collected.(json|html) files * Fix '1 test's'' editorial error on review page * Fix for presenting .collected.html when needed * Add support for multiple settings with .collected * Don't show assertionPhrase column on review page for v1 build * Fix when commandListSettingsPreface gets appended * Change from {at}-{settings} .collected files to a single {at} .collected file which covers all settings * Fix combined .collected.(json|logic) * Update results collection form to support pass/fail assertion verdicts (#1003) * Initial rendering pass * Initial interaction with assertion checkbox * Cleanup and use assertionResponseQuestion * Remove unused required validation, Ensure input fires state change event * Use assertionResponseQuestion in v1 tests as well --------- Co-authored-by: Matt King Co-authored-by: Stalgia Grigg Co-authored-by: Stalgia Grigg --- .../command-tuples-at-mode-task-lookup.js | 26 +- lib/data/parse-command-csv-row.js | 192 ++- lib/data/parse-support.js | 2 + lib/data/parse-test-csv-row.js | 177 ++- lib/data/process-test-directory-v1.js | 1386 +++++++++++++++++ lib/data/process-test-directory.js | 1173 ++++++++------ package.json | 4 +- scripts/create-all-tests.js | 63 +- scripts/review-index-template.mustache | 8 +- scripts/review-template.mustache | 103 +- scripts/test-reviewer.mjs | 499 +++++- scripts/v2maker.js | 115 +- scripts/v2makerUtil.js | 17 +- tests/resources/aria-at-harness.mjs | 63 +- tests/resources/aria-at-test-io-format.mjs | 432 ++++- tests/resources/aria-at-test-run.mjs | 190 +-- tests/resources/at-commands.mjs | 214 ++- tests/resources/keys.mjs | 224 +-- tests/support.json | 12 +- types/aria-at-csv.js | 72 +- types/aria-at-file.js | 2 + types/aria-at-parsed.js | 8 +- 22 files changed, 4025 insertions(+), 957 deletions(-) create mode 100644 lib/data/process-test-directory-v1.js diff --git a/lib/data/command-tuples-at-mode-task-lookup.js b/lib/data/command-tuples-at-mode-task-lookup.js index b8a54f2dc..625dcb7f6 100644 --- a/lib/data/command-tuples-at-mode-task-lookup.js +++ b/lib/data/command-tuples-at-mode-task-lookup.js @@ -31,4 +31,28 @@ function createCommandTuplesATModeTaskLookup(commands) { return data; } -exports.createCommandTuplesATModeTaskLookup = createCommandTuplesATModeTaskLookup; +function createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber(commands) { + return commands.reduce((carry, command) => { + const commandTask = carry[command.testId] || {}; + const commandTaskMode = commandTask[command.target.at.settings || 'defaultMode'] || {}; + const commandTaskModeAT = commandTaskMode[command.target.at.key] || []; + const commandTuples = command.commands.map(({ id, presentationNumber }) => [ + `${id}|${presentationNumber}`, + ]); + return { + ...carry, + [command.testId]: { + ...commandTask, + [command.target.at.settings || 'defaultMode']: { + ...commandTaskMode, + [command.target.at.key]: [...commandTaskModeAT, ...commandTuples], + }, + }, + }; + }, {}); +} + +module.exports = { + createCommandTuplesATModeTaskLookup, + createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber, +}; diff --git a/lib/data/parse-command-csv-row.js b/lib/data/parse-command-csv-row.js index 6cc5c1e2c..0265cfa65 100644 --- a/lib/data/parse-command-csv-row.js +++ b/lib/data/parse-command-csv-row.js @@ -67,4 +67,194 @@ function parseCommandCSVRow(commandRow) { }; } -exports.parseCommandCSVRow = parseCommandCSVRow; +function flattenObject(obj, parentKey = '') { + const flattened = {}; + + for (const key in obj) { + if (typeof obj[key] === 'object') { + const subObject = flattenObject(obj[key], parentKey + key + '.'); + Object.assign(flattened, subObject); + } else { + flattened[parentKey + key] = obj[key]; + } + } + + return flattened; +} + +function sanitizeWhitespace(value) { + return value.replace(/\s+/g, ' ').trim(); +} + +function sanitizeCommand(command) { + return { + ...command, + command: sanitizeWhitespace(command.command), + }; +} + +function findValueByKey(keyMappings, keyToFindText) { + const keyToFind = keyToFindText.replace(/\s+/g, ' ').trim(); + const keyMap = Object.keys(keyMappings); + + // Need to specially handle VO modifier key combination + if (keyToFind === 'vo') + return findValuesByKeys(keyMappings, [keyMappings['modifierAliases.vo']])[0]; + + if (keyToFind.includes('modifiers.') || keyToFind.includes('keys.')) { + const parts = keyToFind.split('.'); + const keyToCheck = parts[parts.length - 1]; // value after the '.' + + if (keyMappings[keyToFind]) + return { + value: keyMappings[keyToFind], + key: keyToCheck, + }; + + return null; + } + + for (const key of keyMap) { + const parts = key.split('.'); + const parentKey = parts[0]; + const keyToCheck = parts[parts.length - 1]; // value after the '.' + + if (keyToCheck === keyToFind) { + if (parentKey === 'modifierAliases') { + return findValueByKey(keyMappings, `modifiers.${keyMappings[key]}`); + } else if (parentKey === 'keyAliases') { + return findValueByKey(keyMappings, `keys.${keyMappings[key]}`); + } + + return { + value: keyMappings[key], + key: keyToCheck, + }; + } + } + + // Return null if the key is not found + return null; +} + +function findValuesByKeys(commandsMapping, keysToFind = []) { + const result = []; + + const patternSepWithReplacement = (keyToFind, pattern, replacement) => { + if (keyToFind.includes(pattern)) { + let value = ''; + let validKeys = true; + const keys = keyToFind.split(pattern); + + for (const key of keys) { + const keyResult = findValueByKey(commandsMapping, key); + if (keyResult) value = value ? `${value}${replacement}${keyResult.value}` : keyResult.value; + else validKeys = false; + } + if (validKeys) return { value, key: keyToFind }; + } + + return null; + }; + + const patternSepHandler = keyToFind => { + let value = ''; + + if (keyToFind.includes(' ') && keyToFind.includes('+')) { + const keys = keyToFind.split(' '); + for (let [index, key] of keys.entries()) { + const keyToFindResult = findValueByKey(commandsMapping, key); + if (keyToFindResult) keys[index] = keyToFindResult.value; + if (key.includes('+')) keys[index] = patternSepWithReplacement(key, '+', '+').value; + } + value = keys.join(' then '); + + return { value, key: keyToFind }; + } else if (keyToFind.includes(' ')) return patternSepWithReplacement(keyToFind, ' ', ' then '); + else if (keyToFind.includes('+')) return patternSepWithReplacement(keyToFind, '+', '+'); + }; + + for (const keyToFind of keysToFind) { + if (keyToFind.includes(' ') || keyToFind.includes('+')) { + result.push(patternSepHandler(keyToFind)); + } else { + const keyToFindResult = findValueByKey(commandsMapping, keyToFind); + if (keyToFindResult) result.push(keyToFindResult); + } + } + + return result; +} + +function parseCommandCSVRowV2({ commands }, commandsJson) { + const commandsParsed = []; + const flattenedCommandsJson = flattenObject(commandsJson); + + function createParsedCommand(command, atTargetInfo) { + const commandIds = command.command.split(' '); + const commandKVs = findValuesByKeys(flattenedCommandsJson, [command.command]); + + const assertionExceptions = command.assertionExceptions.length + ? sanitizeWhitespace(command.assertionExceptions).split(' ') + : []; + + const commands = commandKVs.map(e => ({ + id: e.key, + keystroke: e.value, + keypresses: e.value.split(' then ').map((e, index) => ({ + id: commandIds[index], + keystroke: e, + })), + presentationNumber: Number(command.presentationNumber), + assertionExceptions, + })); + + // Also account for 'modeless' AT configurations with 'defaultMode' + const settingsValue = command.settings || 'defaultMode'; + + const foundCommandIndex = commandsParsed.findIndex( + e => + e.testId === command.testId && + e.target.at.key === atTargetInfo.key && + e.target.at.settings === settingsValue && + (e.commands.length + ? e.commands.every( + ({ presentationNumber }) => + parseInt(presentationNumber) === parseInt(command.presentationNumber) + ) + : true) + ); + + // Add new parsed command since none exist + if (foundCommandIndex < 0) { + commandsParsed.push({ + testId: command.testId, + target: { + at: { + ...atTargetInfo, + settings: settingsValue, + }, + }, + commands, + }); + } else { + commandsParsed[foundCommandIndex].commands.push(...commands); + } + } + + const generateCommandsParsed = (commands, key) => { + for (const command of commands) createParsedCommand(command, { key }); + }; + + commands.forEach(command => + generateCommandsParsed(command.commands.map(sanitizeCommand), command.atKey) + ); + + return commandsParsed; +} + +module.exports = { + parseCommandCSVRow, + parseCommandCSVRowV2, + flattenObject, +}; diff --git a/lib/data/parse-support.js b/lib/data/parse-support.js index 5aed27cd4..ed19102e3 100644 --- a/lib/data/parse-support.js +++ b/lib/data/parse-support.js @@ -22,6 +22,8 @@ function parseSupport(supportRaw) { ats: [at], })), ], + testPlanStrings: supportRaw.testPlanStrings, + references: supportRaw.references, }; } diff --git a/lib/data/parse-test-csv-row.js b/lib/data/parse-test-csv-row.js index 66304e200..c0138e46e 100644 --- a/lib/data/parse-test-csv-row.js +++ b/lib/data/parse-test-csv-row.js @@ -84,4 +84,179 @@ function parseTestCSVRow(testRow) { }; } -exports.parseTestCSVRow = parseTestCSVRow; +function isEmptyObject(object) { + return object && Object.keys(object).length === 0; +} + +function parseTestCSVRowV2({ tests, assertions, scripts, commands }) { + const testsParsed = []; + + // Create containers for all potentially found presentationNumbers; empty ones will be filtered out at the end. + // For eg. in the case where an {at-key}-commands.csv is missing. + // This also maintains the order for testRow.presentationNumber + for (let i = 0; i < tests.length; i++) { + testsParsed.push({}); + } + + function createParsedTest(arrPosition, test, command, atTargetInfo) { + // Create setupScript object + const setupScript = test.setupScript + ? scripts.find(script => test.setupScript === script.setupScript) // { setupScript, setupScriptDescription } + : undefined; + + // Create assertions value + const assertionsValue = test.assertions + ? test.assertions.split(' ').map(assertion => { + // TODO: Return error if foundAssertion undefined + const foundAssertion = assertions.find(e => e.assertionId === assertion); + return foundAssertion + ? { ...foundAssertion, priority: Number(foundAssertion.priority) } + : {}; + }) + : undefined; + + // Create references values + let referencesValue = assertionsValue + ? assertionsValue.flatMap(assertion => assertion.refIds.trim().split(' ')) + : []; + referencesValue = referencesValue.filter(e => e !== ''); + + if (referencesValue.length) { + // Remove duplicates + referencesValue = [...new Set(referencesValue)]; + + // Format to [ { refId: '' } ] + referencesValue = referencesValue.map(refId => { + return { refId }; + }); + } + + // Also account for 'modeless' AT configurations with 'defaultMode' + // TODO: Account for a list of settings as described by https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#settings + const settingsValue = command.settings || 'defaultMode'; + + if (isEmptyObject(testsParsed[arrPosition])) { + testsParsed[arrPosition] = { + testId: test.testId, + title: test.title, + references: referencesValue, + presentationNumber: Number(test.presentationNumber), + target: { + ats: [ + { + ...atTargetInfo, + settings: settingsValue, + }, + ], + }, + setupScript: setupScript + ? { + script: setupScript.setupScript, + scriptDescription: setupScript.setupScriptDescription, + } + : setupScript, + instructions: test.instructions, + assertions: assertionsValue, + }; + } else { + if ( + !testsParsed[arrPosition].target.ats.find( + e => e.key === atTargetInfo.key && e.settings === settingsValue + ) + ) + testsParsed[arrPosition].target.ats.push({ + ...atTargetInfo, + settings: settingsValue, + }); + } + } + + const generateTestsParsed = (commands, key) => { + for (const command of commands) { + const findTest = test => test.testId === command.testId; + + const test = tests.find(findTest); + const testIndex = tests.findIndex(findTest); + createParsedTest(testIndex, test, command, { key }); + } + }; + + const attachCommandInfo = (commands, key) => { + function findCommandInfo({ command, presentationNumber, settings, testId }, commandInfo) { + return ( + testId === commandInfo.testId && + command === commandInfo.command && + settings === commandInfo.settings && + presentationNumber === commandInfo.presentationNumber + ); + } + + for (const command of commands) { + testsParsed.forEach(test => { + test.assertions?.forEach(assertion => { + if ( + command.testId === test.testId && + test.target.ats.some( + at => at.key === key && at.settings === (command.settings || 'defaultMode') + ) + ) { + if (!assertion.commandInfo) assertion.commandInfo = {}; + if (!assertion.commandInfo[key]) assertion.commandInfo[key] = []; + + const commandInfo = { + testId: command.testId, + command: command.command, + settings: command.settings || 'defaultMode', + presentationNumber: Number(command.presentationNumber), + }; + + // assertion level commandInfo, useful for getting assertionExceptions + const assertionCommandInfoExists = assertion.commandInfo[key].find(each => + findCommandInfo(each, commandInfo) + ); + if (!assertionCommandInfoExists) { + const assertionCommandInfo = { + ...commandInfo, + assertionExceptions: command.assertionExceptions, + }; + assertion.commandInfo[key].push(assertionCommandInfo); + } + + // Test level commandInfo, useful for getting review level information + if (!test.commandsInfo) test.commandsInfo = {}; + if (!test.commandsInfo[key]) test.commandsInfo[key] = []; + const testCommandInfoExists = test.commandsInfo[key].find(each => + findCommandInfo(each, commandInfo) + ); + + if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo); + } + }); + }); + } + }; + + // Maintain the order of commands by commandRow.presentationNumber + const commandsSort = (a, b) => + parseFloat(a.presentationNumber) - parseFloat(b.presentationNumber); + + /*const GENERATE_TESTS = [ + [jawsCommands, 'jaws'], + [nvdaCommands, 'nvda'], + [voCommands, 'voiceover_macos'], + ];*/ + const GENERATE_TESTS = commands.map(command => [command.commands, command.atKey]); + GENERATE_TESTS.forEach(([commands, key]) => { + if (commands.length) generateTestsParsed(commands.sort(commandsSort), key); + }); + GENERATE_TESTS.forEach(([commands, key]) => { + if (commands.length) attachCommandInfo(commands.sort(commandsSort), key); + }); + + return testsParsed.filter(testParsed => Object.keys(testParsed).length); +} + +module.exports = { + parseTestCSVRow, + parseTestCSVRowV2, +}; diff --git a/lib/data/process-test-directory-v1.js b/lib/data/process-test-directory-v1.js new file mode 100644 index 000000000..c286e2c78 --- /dev/null +++ b/lib/data/process-test-directory-v1.js @@ -0,0 +1,1386 @@ +/// +/// +/// +/// +/// + +'use strict'; + +const path = require('path'); +const { Readable } = require('stream'); +const { + types: { isArrayBufferView, isArrayBuffer }, +} = require('util'); + +const csv = require('csv-parser'); +const fse = require('fs-extra'); +const beautify = require('json-beautify'); + +const { validate } = require('../util/error'); +const { reindent } = require('../util/lines'); +const { Queryable } = require('../util/queryable'); +const { FileRecordChain } = require('../util/file-record-chain'); + +const { parseSupport } = require('./parse-support'); +const { parseTestCSVRow } = require('./parse-test-csv-row'); +const { parseCommandCSVRow } = require('./parse-command-csv-row'); +const { createCommandTuplesATModeTaskLookup } = require('./command-tuples-at-mode-task-lookup'); + +const { renderHTML: renderCollectedTestHtml } = require('./templates/collected-test.html'); +const { createExampleScriptsTemplate } = require('./example-scripts-template'); + +/** + * @param {string} directory - path to directory of data to be used to generate test + * @param {object} args={} + */ +const processTestDirectory = async ({ directory, args = {} }) => { + let VERBOSE_CHECK = false; + let VALIDATE_CHECK = false; + + let suppressedMessages = 0; + + /** + * @param {string} message - message to be logged + * @param {object} [options] + * @param {boolean} [options.severe=false] - indicates whether the message should be viewed as an error or not + * @param {boolean} [options.force=false] - indicates whether this message should be forced to be outputted regardless of verbosity level + */ + const log = (message, { severe = false, force = false } = {}) => { + if (VERBOSE_CHECK || force) { + if (severe) console.error(message); + else console.log(message); + } else { + // Output no logs + suppressedMessages += 1; // counter to indicate how many messages were hidden + } + }; + + /** + * @param {string} message - error message + */ + log.warning = message => log(message, { severe: true, force: true }); + + /** + * Log error then exit the process. + * @param {string} message - error message + */ + log.error = message => { + log.warning(message); + process.exit(1); + }; + + // setup from arguments passed to npm script + VERBOSE_CHECK = !!args.verbose; + VALIDATE_CHECK = !!args.validate; + + const validModes = ['reading', 'interaction', 'item']; + + // cwd; @param rootDirectory is dependent on this file not moving from the `lib/data` folder + const libDataDirectory = path.dirname(__filename); + const rootDirectory = path.join(libDataDirectory, '../..'); + + const testsDirectory = path.join(rootDirectory, 'tests'); + const testPlanDirectory = path.join(rootDirectory, directory); + + const resourcesDirectory = path.join(testsDirectory, 'resources'); + + const supportFilePath = path.join(testsDirectory, 'support.json'); + const atCommandsCsvFilePath = path.join(testPlanDirectory, 'data', 'commands.csv'); + const testsCsvFilePath = path.join(testPlanDirectory, 'data', 'testsV1.csv'); + const referencesCsvFilePath = path.join(testPlanDirectory, 'data', 'referencesV1.csv'); + + // build output folders and file paths setup + const buildDirectory = path.join(rootDirectory, 'build'); + const testPlanBuildDirectory = path.join(buildDirectory, directory); + const indexFileBuildOutputPath = path.join(testPlanBuildDirectory, 'index.html'); + + let backupTestsCsvFile, backupReferencesCsvFile; + + // create build directory if it doesn't exist + fse.mkdirSync(buildDirectory, { recursive: true }); + + const existingBuildPromise = FileRecordChain.read(buildDirectory, { + glob: [ + '', + 'tests', + `tests/${path.basename(directory)}`, + `tests/${path.basename(directory)}/**`, + 'tests/resources', + 'tests/resources/*', + 'tests/support.json', + ].join(','), + }); + + const [testPlanRecord, resourcesOriginalRecord, supportRecord] = await Promise.all( + [testPlanDirectory, resourcesDirectory, supportFilePath].map(filepath => + FileRecordChain.read(filepath) + ) + ); + + const scriptsRecord = testPlanRecord.find('data/js'); + const resourcesRecord = resourcesOriginalRecord.filter({ glob: '{aria-at-*,keys,vrender}.mjs' }); + + const newBuild = new FileRecordChain({ + entries: [ + { + name: 'tests', + entries: [ + { + name: path.basename(directory), + entries: testPlanRecord.filter({ glob: 'reference{,/**}' }).record.entries, + }, + { name: 'resources', ...resourcesRecord.record }, + { name: 'support.json', ...supportRecord.record }, + ], + }, + ], + }); + + const keyDefs = {}; + const supportJson = JSON.parse(supportRecord.text); + + let allATKeys = supportJson.ats.map(({ key }) => key); + let allATNames = supportJson.ats.map(({ name }) => name); + + const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys); + + if (!testPlanRecord.isDirectory()) { + log.error(`The test directory '${testPlanDirectory}' does not exist. Check the path to tests.`); + } + + if (!testPlanRecord.find('data/commands.csv').isFile()) { + log.error( + `The at-commands.csv file does not exist. Please create '${atCommandsCsvFilePath}' file.` + ); + } + + if (!testPlanRecord.find('data/testsV1.csv').isFile()) { + // Check if original file can be processed + if (!testPlanRecord.find('data/tests.csv').isFile()) { + log.error(`The testsV1.csv file does not exist. Please create '${testsCsvFilePath}' file.`); + } else { + backupTestsCsvFile = 'data/tests.csv'; + } + } + + if (!testPlanRecord.find('data/referencesV1.csv').isFile()) { + // Check if original file can be processed + if (!testPlanRecord.find('data/references.csv').isFile()) { + log.error( + `The referencesV1.csv file does not exist. Please create '${referencesCsvFilePath}' file.` + ); + } else { + backupReferencesCsvFile = 'data/references.csv'; + } + } + + // get Keys that are defined + try { + // read contents of the file + const keys = resourcesRecord.find('keys.mjs').text; + + // split the contents by new line + const lines = keys.split(/\r?\n/); + + // print all lines + lines.forEach(line => { + let parts1 = line.split(' '); + let parts2 = line.split('"'); + + if (parts1.length > 3) { + let code = parts1[2].trim(); + keyDefs[code] = parts2[1].trim(); + } + }); + } catch (err) { + log.warning(err); + } + + function cleanTask(task) { + return task.replace(/'/g, '').replace(/;/g, '').trim().toLowerCase(); + } + + /** + * Create Test File + * @param {AriaATCSV.Test} test + * @param refs + * @param commands + * @returns {(string|*[])[]} + */ + function createTestFile( + test, + refs, + commands, + { addTestError, emitFile, scriptsRecord, exampleScriptedFilesQueryable } + ) { + let scripts = []; + + // default setupScript if test has undefined setupScript + if (!scriptsRecord.find(`${test.setupScript}.js`).isFile()) test.setupScript = ''; + + function getModeValue(value) { + let v = value.trim().toLowerCase(); + if (!validModes.includes(v)) { + addTestError(test.testId, '"' + value + '" is not valid value for "mode" property.'); + } + return v; + } + + function getTask(t) { + let task = cleanTask(t); + + if (typeof commands[task] !== 'object') { + addTestError(test.testId, '"' + task + '" does not exist in commands.csv file.'); + } + + return task; + } + + function getAppliesToValues(values) { + function checkValue(value) { + let v1 = value.trim().toLowerCase(); + for (let i = 0; i < validAppliesTo.length; i++) { + let v2 = validAppliesTo[i]; + if (v1 === v2.toLowerCase()) { + return v2; + } + } + return false; + } + + // check for individual assistive technologies + let items = values.split(','); + let newValues = []; + items.filter(item => { + let value = checkValue(item); + if (!value) { + addTestError(test.testId, '"' + item + '" is not valid value for "appliesTo" property.'); + } + + newValues.push(value); + }); + + return newValues; + } + + /** + * Determines priority level (default is 1) of assertion string, then adds it to the collection of assertions for + * the test plan + * @param {string} a - Assertion string to be evaluated + */ + function addAssertion(a) { + let level = '1'; + let str = a; + a = a.trim(); + + // matches a 'colon' when preceded by either of the digits 1 OR 2 (SINGLE CHARACTER), at the start of the string + let parts = a.split(/(?<=^[1-2]):/g); + + if (parts.length === 2) { + level = parts[0]; + str = parts[1].substring(0); + if (level !== '1' && level !== '2') { + addTestError( + test.testId, + "Level value must be 1 or 2, value found was '" + + level + + "' for assertion '" + + str + + "' (NOTE: level 2 defined for this assertion)." + ); + level = '2'; + } + } + + if (a.length) { + assertions.push([level, str]); + } + } + + function getReferences(example, testRefs) { + let links = ''; + + if (typeof example === 'string' && example.length) { + links += `\n`; + } + + let items = test.refs.split(' '); + items.forEach(function (item) { + item = item.trim(); + + if (item.length) { + if (typeof refs[item] === 'string') { + links += `\n`; + } else { + addTestError(test.testId, 'Reference does not exist: ' + item); + } + } + }); + + return links; + } + + function addSetupScript(scriptName) { + let script = ''; + if (scriptName) { + if (!scriptsRecord.find(`${scriptName}.js`).isFile()) { + addTestError(test.testId, `Setup script does not exist: ${scriptName}.js`); + return ''; + } + + try { + const data = scriptsRecord.find(`${scriptName}.js`).text; + const lines = data.split(/\r?\n/); + lines.forEach(line => { + if (line.trim().length) script += '\t\t\t' + line.trim() + '\n'; + }); + } catch (err) { + log.warning(err); + } + + scripts.push(`\t\t${scriptName}: function(testPageDocument){\n${script}\t\t}`); + } + + return script; + } + + function getSetupScriptDescription(desc) { + let str = ''; + if (typeof desc === 'string') { + let d = desc.trim(); + if (d.length) { + str = d; + } + } + + return str; + } + + function getScripts() { + let js = 'var scripts = {\n'; + js += scripts.join(',\n'); + js += '\n\t};'; + return js; + } + + let task = getTask(test.task); + let appliesTo = getAppliesToValues(test.appliesTo); + let mode = getModeValue(test.mode); + + appliesTo.forEach(at => { + if (commands[task]) { + if (!commands[task][mode][at.toLowerCase()]) { + addTestError( + test.testId, + 'command is missing for the combination of task: "' + + task + + '", mode: "' + + mode + + '", and AT: "' + + at.toLowerCase() + + '" ' + ); + } + } + }); + + let assertions = []; + let id = test.testId; + if (parseInt(test.testId) < 10) { + id = '0' + id; + } + + const cleanTaskName = cleanTask(test.task).replace(/\s+/g, '-'); + let testFileName = `test-${id}-${cleanTaskName}-${mode}.html`; + let testJSONFileName = `test-${id}-${cleanTaskName}-${mode}.json`; + + let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, testFileName); + let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, testJSONFileName); + + let references = getReferences(refs.example, test.refs); + addSetupScript(test.setupScript); + + for (let i = 1; i < 31; i++) { + if (!test['assertion' + i]) { + continue; + } + addAssertion(test['assertion' + i]); + } + + /** @type {AriaATFile.Behavior} */ + let testData = { + setup_script_description: getSetupScriptDescription(test.setupScriptDescription), + setupTestPage: test.setupScript, + applies_to: appliesTo, + mode: mode, + task: task, + assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion, + specific_user_instruction: test.instructions, + output_assertions: assertions, + }; + + emitFile(testPlanJsonFileBuildPath, JSON.stringify(testData, null, 2), 'utf8'); + + function getTestJson() { + return JSON.stringify(testData, null, 2); + } + + function getCommandsJson() { + return beautify({ [task]: commands[task] }, null, 2, 40); + } + + let testHTML = ` + + +${test.title} +${references} + + + `; + + emitFile(testPlanHtmlFileBuildPath, testHTML, 'utf8'); + + /** @type {AriaATFile.CollectedTest} */ + const collectedTest = {}; + + const applies_to_at = []; + + allATKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0)); + + return [testFileName, applies_to_at]; + } + + /** + * Create an index file for a local server + * @param tasks + */ + function createIndexFile(tasks, { emitFile }) { + let rows = ''; + let all_ats = ''; + + allATNames.forEach(at => (all_ats += '' + at + '\n')); + + tasks.forEach(function (task) { + rows += `${task.id}`; + rows += `${task.title}`; + for (let i = 0; i < allATKeys.length; i++) { + if (task.applies_to_at[i]) { + rows += `${allATNames[i]}`; + } else { + rows += `not included`; + } + } + rows += `${task.script}\n`; + }); + + let indexHTML = ` + + + + Index of Assistive Technology Test Files + + + +
+

Index of Assistive Technology Test Files

+

This is useful for viewing the local files on a local web server.

+ + + + + + ${all_ats} + + + + +${rows} + +
Task IDTesting TaskSetup Script Reference
+
+ +`; + + emitFile(indexFileBuildOutputPath, indexHTML, 'utf8'); + } + + // Process CSV files + var refs = {}; + var errorCount = 0; + var errors = ''; + var indexOfURLs = []; + var checkedSourceHtmlScriptFiles = []; + + function addTestError(id, error) { + errorCount += 1; + errors += '[Test ' + id + ']: ' + error + '\n'; + } + + function addCommandError({ testId, task }, key) { + errorCount += 1; + errors += `[Command]: The key reference "${key}" found in "${directory}/data/commands.csv" for "test id ${testId}: ${task}" is invalid. Command may not be defined in "tests/resources/keys.mjs".\n`; + } + + const newTestPlan = newBuild.find(`tests/${path.basename(testPlanBuildDirectory)}`); + function emitFile(filepath, content) { + newTestPlan.add(path.relative(testPlanBuildDirectory, filepath), { + buffer: toBuffer(content), + }); + } + + function generateSourceHtmlScriptFile(filePath, content) { + // check that test plan's reference html file path is generated file + if ( + filePath.includes('reference') && + (filePath.split(path.sep).pop().match(/\./g) || []).length > 1 + ) { + // generate file at `/tests//reference//. +`, + button: reindent` + +
+ +
+`, + }; +} diff --git a/lib/data/process-test-directory.js b/lib/data/process-test-directory.js index 9e3d76110..d27079037 100644 --- a/lib/data/process-test-directory.js +++ b/lib/data/process-test-directory.js @@ -22,9 +22,11 @@ const { Queryable } = require('../util/queryable'); const { FileRecordChain } = require('../util/file-record-chain'); const { parseSupport } = require('./parse-support'); -const { parseTestCSVRow } = require('./parse-test-csv-row'); -const { parseCommandCSVRow } = require('./parse-command-csv-row'); -const { createCommandTuplesATModeTaskLookup } = require('./command-tuples-at-mode-task-lookup'); +const { parseTestCSVRowV2 } = require('./parse-test-csv-row'); +const { parseCommandCSVRowV2, flattenObject } = require('./parse-command-csv-row'); +const { + createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber, +} = require('./command-tuples-at-mode-task-lookup'); const { renderHTML: renderCollectedTestHtml } = require('./templates/collected-test.html'); const { createExampleScriptsTemplate } = require('./example-scripts-template'); @@ -34,9 +36,6 @@ const { createExampleScriptsTemplate } = require('./example-scripts-template'); * @param {object} args={} */ const processTestDirectory = async ({ directory, args = {} }) => { - let VERBOSE_CHECK = false; - let VALIDATE_CHECK = false; - let suppressedMessages = 0; /** @@ -70,10 +69,8 @@ const processTestDirectory = async ({ directory, args = {} }) => { }; // setup from arguments passed to npm script - VERBOSE_CHECK = !!args.verbose; - VALIDATE_CHECK = !!args.validate; - - const validModes = ['reading', 'interaction', 'item']; + const VERBOSE_CHECK = !!args.verbose; + const VALIDATE_CHECK = !!args.validate; // cwd; @param rootDirectory is dependent on this file not moving from the `lib/data` folder const libDataDirectory = path.dirname(__filename); @@ -83,11 +80,12 @@ const processTestDirectory = async ({ directory, args = {} }) => { const testPlanDirectory = path.join(rootDirectory, directory); const resourcesDirectory = path.join(testsDirectory, 'resources'); - - const supportFilePath = path.join(testsDirectory, 'support.json'); + const supportJsonFilePath = path.join(testsDirectory, 'support.json'); + const commandsJsonFilePath = path.join(testsDirectory, 'commands.json'); const testsCsvFilePath = path.join(testPlanDirectory, 'data', 'tests.csv'); - const atCommandsCsvFilePath = path.join(testPlanDirectory, 'data', 'commands.csv'); const referencesCsvFilePath = path.join(testPlanDirectory, 'data', 'references.csv'); + const assertionsCsvFilePath = path.join(testPlanDirectory, 'data', 'assertions.csv'); + const scriptsCsvFilePath = path.join(testPlanDirectory, 'data', 'scripts.csv'); // build output folders and file paths setup const buildDirectory = path.join(rootDirectory, 'build'); @@ -106,14 +104,16 @@ const processTestDirectory = async ({ directory, args = {} }) => { 'tests/resources', 'tests/resources/*', 'tests/support.json', + 'tests/commands.json', ].join(','), }); - const [testPlanRecord, resourcesOriginalRecord, supportRecord] = await Promise.all( - [testPlanDirectory, resourcesDirectory, supportFilePath].map(filepath => - FileRecordChain.read(filepath) - ) - ); + const [testPlanRecord, resourcesOriginalRecord, supportJsonRecord, commandsJsonRecord] = + await Promise.all( + [testPlanDirectory, resourcesDirectory, supportJsonFilePath, commandsJsonFilePath].map( + filepath => FileRecordChain.read(filepath) + ) + ); const scriptsRecord = testPlanRecord.find('data/js'); const resourcesRecord = resourcesOriginalRecord.filter({ glob: '{aria-at-*,keys,vrender}.mjs' }); @@ -128,19 +128,33 @@ const processTestDirectory = async ({ directory, args = {} }) => { entries: testPlanRecord.filter({ glob: 'reference{,/**}' }).record.entries, }, { name: 'resources', ...resourcesRecord.record }, - { name: 'support.json', ...supportRecord.record }, + { name: 'support.json', ...supportJsonRecord.record }, + { name: 'commands.json', ...commandsJsonRecord.record }, ], }, ], }); - const keyDefs = {}; - const support = JSON.parse(supportRecord.text); + const supportJson = JSON.parse(supportJsonRecord.text); + const commandsJson = JSON.parse(commandsJsonRecord.text); - let allATKeys = support.ats.map(({ key }) => key); - let allATNames = support.ats.map(({ name }) => name); + const allAts = supportJson.ats; + const allAtKeys = allAts.map(({ key }) => key); + const allAtNames = allAts.map(({ name }) => name); + const atSettings = allAts + .map(({ settings }) => settings) + .flatMap(setting => Object.keys(setting)); - const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys); + // Get paths to possible at keys + const atCommandsCsvFilePaths = getTestPlanDataFilePaths( + testPlanDirectory, + allAtKeys.map(key => `${key}-commands.csv`) + ).map((atCommandCsvFilePath, index) => ({ atKey: allAtKeys[index], atCommandCsvFilePath })); + + // readingMode and interactionMode are known screen reader 'at modes' found in + // support.json at ats[].assertionTokens. The specific named modes are + // stored in ats[].settings + const validModes = ['readingMode', 'interactionMode', 'defaultMode'].concat(atSettings); if (!testPlanRecord.isDirectory()) { log.error(`The test directory '${testPlanDirectory}' does not exist. Check the path to tests.`); @@ -150,158 +164,285 @@ const processTestDirectory = async ({ directory, args = {} }) => { log.error(`The tests.csv file does not exist. Please create '${testsCsvFilePath}' file.`); } - if (!testPlanRecord.find('data/commands.csv').isFile()) { + if (!testPlanRecord.find('data/references.csv').isFile()) { log.error( - `The at-commands.csv file does not exist. Please create '${atCommandsCsvFilePath}' file.` + `The references.csv file does not exist. Please create '${referencesCsvFilePath}' file.` ); } - if (!testPlanRecord.find('data/references.csv').isFile()) { + if (!testPlanRecord.find('data/assertions.csv').isFile()) { log.error( - `The references.csv file does not exist. Please create '${referencesCsvFilePath}' file.` + `The assertions.csv file does not exist. Please create '${assertionsCsvFilePath}' file.` ); } - // get Keys that are defined - try { - // read contents of the file - const keys = resourcesRecord.find('keys.mjs').text; + if (!testPlanRecord.find('data/scripts.csv').isFile()) { + log.error(`The scripts.csv file does not exist. Please create '${scriptsCsvFilePath}' file.`); + } + + let someAtCommandsCsvExist = false; + let missingAtCommandsCsvFiles = []; + for (const atCommandCsvFilePath of atCommandsCsvFilePaths) { + const { atKey, atCommandCsvFilePath: filePath } = atCommandCsvFilePath; - // split the contents by new line - const lines = keys.split(/\r?\n/); + // Checks that at least one {at}-commands.csv exists + if (testPlanRecord.find(`data/${atKey}-commands.csv`).isFile()) someAtCommandsCsvExist = true; + else missingAtCommandsCsvFiles.push(filePath); + } + if (!someAtCommandsCsvExist) { + log.error( + `No *-commands.csv files found for ${directory}. Please create at least one of the following: ${missingAtCommandsCsvFiles.join( + ', ' + )}` + ); + } - // print all lines - lines.forEach(line => { - let parts1 = line.split(' '); - let parts2 = line.split('"'); + /** + * @param {AriaATCSV.Reference} row + * @returns {AriaATCSV.Reference} + */ + function validateReferencesKeys(row) { + if ( + typeof row.refId !== 'string' || + typeof row.type !== 'string' || + typeof row.value !== 'string' + ) { + throw new Error('Row missing refId, type or value'); + } + return row; + } - if (parts1.length > 3) { - let code = parts1[2].trim(); - keyDefs[code] = parts2[1].trim(); + const validCommandKeys = /^(?:testId|command|settings|assertionExceptions|presentationNumber)$/; + const numericKeyFormat = /^_(\d+)$/; + const idFormat = /^[A-Za-z0-9-]+$/; + const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-]+\s*)+$/; + const settingsFormat = /^[A-Za-z0-9-\s]+$/; + function validateCommandsKeys(row) { + // example header: + // testId,command,settings,assertionExceptions,presentationNumber + for (const key of Object.keys(row)) { + if (numericKeyFormat.test(key)) { + throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); + } else if (!validCommandKeys.test(key)) { + throw new Error(`Unknown *-commands.csv key: ${key} - check header row?`); } - }); - } catch (err) { - log.warning(err); + } + if (!(row.testId?.length && row.command?.length && row.presentationNumber?.length)) { + throw new Error('Missing one of required testId, command, presentationNumber'); + } + if (!idFormat.test(row.testId)) + throw new Error('testId does not match the expected format: ' + row.testId); + if (row.settings && !settingsFormat.test(row.settings)) + throw new Error('settings does not match the expected format: ' + row.settings); + if (row.assertionExceptions && !assertionsExceptionsFormat.test(row.assertionExceptions)) + throw new Error( + 'assertionExceptions does not match the expectedFormat: ' + row.assertionExceptions + ); + if (!Number(row.presentationNumber) > 0) + throw new Error( + 'presentationNumber does not match the expected format: ' + row.presentationNumber + ); + return row; } - function cleanTask(task) { - return task.replace(/'/g, '').replace(/;/g, '').trim().toLowerCase(); + const validTestsKeys = + /^(?:testId|title|presentationNumber|setupScript|instructions|assertions)$/; + const titleFormat = /^[A-Z]([A-Za-z-',\s]){2,}[^.]$/; + function validateTestsKeys(row) { + // example header: + // testId,title,presentationNumber,setupScript,instructions,assertions + for (const key of Object.keys(row)) { + if (numericKeyFormat.test(key)) { + throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); + } else if (!validTestsKeys.test(key)) { + throw new Error(`Unknown tests.csv key: ${key} - check header row?`); + } + } + if ( + !( + row.testId?.length && + row.title?.length && + row.presentationNumber?.length && + row.instructions?.length && + row.assertions?.length + ) + ) { + throw new Error( + 'Missing one of required testId, title, presentationNumber, instructions, assertions' + ); + } + if (!idFormat.test(row.testId)) + throw new Error('testId does not match the expected format: ' + row.testId); + if (!titleFormat.test(row.title)) + throw new Error('title does not match the expected format: ' + row.title); + if (!Number(row.presentationNumber) > 0) + throw new Error( + 'presentationNumber does not match the expected format: ' + row.presentationNumber + ); + return row; + } + + const validAssertionsKeys = + /^(?:assertionId|priority|assertionStatement|assertionPhrase|refIds)$/; + const priorityFormat = /^[123]$/; + // TODO: The make-v2 script isn't removing the punctuations at the end which violates the requirements of this + // column's validation. Based on https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#assertionstatement + // Please ensure they are removing and use the commented regex instead. + const assertionStatementFormat = /^[A-Z].*[a-zA-Z'{}().](?![.!?])$/; + // const assertionStatementFormat = /^[A-Z].*[a-zA-Z'{}()](?![.!?])$/; + + // TODO: Same as TODO above. Based on https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#assertionphrase + const assertionPhraseFormat = /^[a-z].*[a-zA-Z'"{}()\d+.](?![.!?])$/; + // const assertionPhraseFormat = /^[a-z].*[a-zA-Z'"{}()\d+](?![.!?])$/; + function validateAssertionsKeys(row) { + // example header: + // assertionId,priority,assertionStatement,assertionPhrase,refIds + for (const key of Object.keys(row)) { + if (numericKeyFormat.test(key)) { + throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); + } else if (!validAssertionsKeys.test(key)) { + throw new Error(`Unknown tests.csv key: ${key} - check header row?`); + } + } + if ( + !( + row.assertionId?.length && + row.priority?.length && + row.assertionStatement?.length && + row.assertionPhrase?.length + ) + ) { + throw new Error( + 'Missing one of required assertionId, priority, assertionStatement, assertionPhrase' + ); + } + if (!idFormat.test(row.assertionId)) + throw new Error('assertionId does not match the expected format: ' + row.assertionId); + if (!priorityFormat.test(row.priority)) + throw new Error('priority does not match the expected format: ' + row.priority); + if (!assertionStatementFormat.test(row.assertionStatement)) + throw new Error( + 'assertionStatement does not match the expected format: ' + row.assertionStatement + ); + if (!assertionPhraseFormat.test(row.assertionPhrase)) + throw new Error('assertionPhrase does not match the expected format: ' + row.assertionPhrase); + return row; + } + + const validScriptsKeys = /^(?:setupScript|setupScriptDescription)$/; + const setupScriptDescriptionFormat = /^[a-z].*[a-zA-Z'](?![.!?])$/; + function validateScriptsKeys(row) { + // example header: + // setupScript,setupScriptDescription + for (const key of Object.keys(row)) { + if (numericKeyFormat.test(key)) { + throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); + } else if (!validScriptsKeys.test(key)) { + throw new Error(`Unknown tests.csv key: ${key} - check header row?`); + } + } + if (!(row.setupScript?.length && row.setupScriptDescription?.length)) { + throw new Error('Missing one of required setupScript, setupScriptDescription'); + } + if (!setupScriptDescriptionFormat.test(row.setupScriptDescription)) + throw new Error( + 'setupScriptDescription does not match the expected format: ' + row.setupScriptDescription + ); + return row; + } + + const [referencesCsv, testsCsv, assertionsCsv, scriptsCsv, ...atCommandsCsvs] = await Promise.all( + [ + readCSVFile('data/references.csv', validateReferencesKeys), + readCSVFile('data/tests.csv', validateTestsKeys), + readCSVFile('data/assertions.csv', validateAssertionsKeys), + readCSVFile('data/scripts.csv', validateScriptsKeys), + ...atCommandsCsvFilePaths.map(({ atKey }) => + readCSVFile(`data/${atKey}-commands.csv`, validateCommandsKeys) + ), + ] + ); + + // Works because the order of allAtKeys and atCommandsCsvs should be maintained up this point + const parsedAtCommandsCsvs = allAtKeys.map((atKey, index) => ({ + atKey, + commands: atCommandsCsvs[index], + })); + + const testsParsed = parseTestCSVRowV2({ + tests: testsCsv, + assertions: assertionsCsv, + scripts: scriptsCsv, + commands: parsedAtCommandsCsvs, + }); + + /** + * + * @param {string} root + * @param {string[]} fileNames + * @return {array[string]} + */ + function getTestPlanDataFilePaths(root, fileNames = []) { + let filePaths = []; + + for (const fileName of fileNames) { + const filePath = path.join(testPlanDirectory, 'data', fileName); + filePaths.push(filePath); + } + + return filePaths; } /** * Create Test File * @param {AriaATCSV.Test} test - * @param refs - * @param commands + * @param {Object>} refs + * @param {Object>>} atCommandsMap + * @param {object} options + * @param {function(string, string)} options.addTestError + * @param {function(filePath: string, content: any, encoding: string)} options.emitFile + * @param {FileRecordChain} options.scriptsRecord + * @param {Queryable} options.exampleScriptedFilesQueryable * @returns {(string|*[])[]} */ function createTestFile( test, refs, - commands, + atCommandsMap, { addTestError, emitFile, scriptsRecord, exampleScriptedFilesQueryable } ) { let scripts = []; // default setupScript if test has undefined setupScript - if (!scriptsRecord.find(`${test.setupScript}.js`).isFile()) test.setupScript = ''; - - function getModeValue(value) { - let v = value.trim().toLowerCase(); - if (!validModes.includes(v)) { - addTestError(test.testId, '"' + value + '" is not valid value for "mode" property.'); - } - return v; - } - - function getTask(t) { - let task = cleanTask(t); - - if (typeof commands[task] !== 'object') { - addTestError(test.testId, '"' + task + '" does not exist in commands.csv file.'); - } - - return task; - } + if (!scriptsRecord.find(`${test.setupScript?.script}.js`).isFile()) + test.setupScript = { + ...test.setupScript, + script: '', + }; - function getAppliesToValues(values) { - function checkValue(value) { - let v1 = value.trim().toLowerCase(); - for (let i = 0; i < validAppliesTo.length; i++) { - let v2 = validAppliesTo[i]; - if (v1 === v2.toLowerCase()) { - return v2; - } - } - return false; + function getTestId(testId) { + if (typeof atCommandsMap[testId] !== 'object') { + addTestError(test.testId, '"' + testId + '" does not exist in *-commands.csv file.'); } - - // check for individual assistive technologies - let items = values.split(','); - let newValues = []; - items.filter(item => { - let value = checkValue(item); - if (!value) { - addTestError(test.testId, '"' + item + '" is not valid value for "appliesTo" property.'); - } - - newValues.push(value); - }); - - return newValues; + return testId; } - /** - * Determines priority level (default is 1) of assertion string, then adds it to the collection of assertions for - * the test plan - * @param {string} a - Assertion string to be evaluated - */ - function addAssertion(a) { - let level = '1'; - let str = a; - a = a.trim(); - - // matches a 'colon' when preceded by either of the digits 1 OR 2 (SINGLE CHARACTER), at the start of the string - let parts = a.split(/(?<=^[1-2]):/g); - - if (parts.length === 2) { - level = parts[0]; - str = parts[1].substring(0); - if (level !== '1' && level !== '2') { - addTestError( - test.testId, - "Level value must be 1 or 2, value found was '" + - level + - "' for assertion '" + - str + - "' (NOTE: level 2 defined for this assertion)." - ); - level = '2'; - } - } - - if (a.length) { - assertions.push([level, str]); + function getModeValue({ settings }) { + let v = settings.trim(); + if (!validModes.includes(v)) { + addTestError(test.testId, '"' + settings + '" is not valid value for "settings" property.'); } + return v; } - function getReferences(example, testRefs) { + function getReferences(test, refs) { let links = ''; - if (typeof example === 'string' && example.length) { - links += `\n`; - } + test.references.forEach(({ refId }) => { + const { value: link, linkText } = refs[refId]; - let items = test.refs.split(' '); - items.forEach(function (item) { - item = item.trim(); - - if (item.length) { - if (typeof refs[item] === 'string') { - links += `\n`; - } else { - addTestError(test.testId, 'Reference does not exist: ' + item); - } + if (typeof link === 'string' && link.length) { + links += `\n`; } }); @@ -345,65 +486,51 @@ const processTestDirectory = async ({ directory, args = {} }) => { } function getScripts() { - let js = 'var scripts = {\n'; + let js = 'let scripts = {\n'; js += scripts.join(',\n'); js += '\n\t};'; return js; } - let task = getTask(test.task); - let appliesTo = getAppliesToValues(test.appliesTo); - let mode = getModeValue(test.mode); + let testId = getTestId(test.testId); + let modes = test.target.ats.map(getModeValue).join('_'); - appliesTo.forEach(at => { - if (commands[task]) { - if (!commands[task][mode][at.toLowerCase()]) { + test.target.ats.forEach(at => { + if (atCommandsMap[testId]) { + if (!atCommandsMap[testId][at.settings][at.key]) { addTestError( - test.testId, + testId, 'command is missing for the combination of task: "' + - task + + testId + '", mode: "' + - mode + + atCommandsMap[testId][at.settings] + '", and AT: "' + - at.toLowerCase() + + at.key.toLowerCase() + '" ' ); } } }); - let assertions = []; - let id = test.testId; - if (parseInt(test.testId) < 10) { - id = '0' + id; - } - - const cleanTaskName = cleanTask(test.task).replace(/\s+/g, '-'); - let testFileName = `test-${id}-${cleanTaskName}-${mode}.html`; - let testJSONFileName = `test-${id}-${cleanTaskName}-${mode}.json`; - - let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, testFileName); - let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, testJSONFileName); + // testId and test level presentationNumber should be enough of a descriptor to differentiate the values + let testFileName = `test-${test.presentationNumber.toString().padStart(2, '0')}-${testId}`; + let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, `${testFileName}.html`); + let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, `${testFileName}.json`); - let references = getReferences(refs.example, test.refs); - addSetupScript(test.setupScript); - - for (let i = 1; i < 31; i++) { - if (!test['assertion' + i]) { - continue; - } - addAssertion(test['assertion' + i]); - } + let exampleReferences = getReferences(test, refs); + addSetupScript(test.setupScript.script); /** @type {AriaATFile.Behavior} */ let testData = { - setup_script_description: getSetupScriptDescription(test.setupScriptDescription), - setupTestPage: test.setupScript, - applies_to: appliesTo, - mode: mode, - task: task, + task: testId, + mode: modes, + applies_to: [...new Set(test.target.ats.map(({ key }) => key))], + setup_script_description: getSetupScriptDescription(test.setupScript.scriptDescription), + setupTestPage: test.setupScript.script, specific_user_instruction: test.instructions, - output_assertions: assertions, + assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion, + commandsInfo: test.commandsInfo, + output_assertions: test.assertions, }; emitFile(testPlanJsonFileBuildPath, JSON.stringify(testData, null, 2), 'utf8'); @@ -413,14 +540,14 @@ const processTestDirectory = async ({ directory, args = {} }) => { } function getCommandsJson() { - return beautify({ [task]: commands[task] }, null, 2, 40); + return beautify({ [testId]: atCommandsMap[testId] }, null, 2, 40); } let testHTML = ` ${test.title} -${references} +${exampleReferences} @@ -430,14 +557,20 @@ ${references} new Promise((resolve) => { fetch('../support.json') .then(response => resolve(response.json())) + }).then(supportJson => { + return fetch('../commands.json') + .then(response => response.json()) + .then(allCommandsJson => ({ supportJson, allCommandsJson }) + ); }) - .then(supportJson => { + .then(({ supportJson, allCommandsJson }) => { const testJson = ${getTestJson()}; const commandJson = ${getCommandsJson()}; - initialize(supportJson, commandJson); + initialize(supportJson, commandJson, allCommandsJson); verifyATBehavior(testJson); displayTestPageAndInstructions(${JSON.stringify( - exampleScriptedFilesQueryable.where({ name: test.setupScript ? test.setupScript : '' }).path + exampleScriptedFilesQueryable.where({ name: test.setupScript ? test.setupScript.script : '' }) + .path )}); }); @@ -445,37 +578,44 @@ ${references} emitFile(testPlanHtmlFileBuildPath, testHTML, 'utf8'); - /** @type {AriaATFile.CollectedTest} */ - const collectedTest = {}; - const applies_to_at = []; - - allATKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0)); + allAtKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0)); return [testFileName, applies_to_at]; } /** * Create an index file for a local server - * @param tasks + * @param {object[]} tasks + * @param {number} tasks.seq + * @param {string} tasks.id + * @param {string} tasks.title + * @param {string} tasks.href + * @param {object} tasks.script + * @param {string} tasks.script.script + * @param {string} tasks.script.scriptDescription + * @param {boolean[]} tasks.applies_to_at + * @param {object} options + * @param {function(filePath: string, content: any, encoding: string)} options.emitFile */ function createIndexFile(tasks, { emitFile }) { let rows = ''; let all_ats = ''; - allATNames.forEach(at => (all_ats += '' + at + '\n')); + allAtNames.forEach(at => (all_ats += '' + at + '\n')); tasks.forEach(function (task) { - rows += `${task.id}`; + rows += `${task.seq}`; + rows += `${task.id}`; rows += `${task.title}`; - for (let i = 0; i < allATKeys.length; i++) { + for (let i = 0; i < allAtKeys.length; i++) { if (task.applies_to_at[i]) { - rows += `${allATNames[i]}`; + rows += `${allAtNames[i]}`; } else { rows += `not included`; } } - rows += `${task.script}\n`; + rows += `${task.script.script}\n`; }); let indexHTML = ` @@ -488,7 +628,7 @@ ${references} display: table; border-collapse: collapse; border-spacing: 2px; - border-color: gray; + border-color: rgb(128,128,128); } thead { @@ -500,7 +640,7 @@ ${references} tbody { display: table-row-group; vertical-align: middle; - border-color: gray; + border-color: rgb(128,128,128); } tr:nth-child(even) {background: #DDD} @@ -509,7 +649,7 @@ ${references} tr { display: table-row; vertical-align: inherit; - border-color: gray; + border-color: rgb(128,128,128); } td { @@ -539,6 +679,7 @@ ${references} + ${all_ats} @@ -557,20 +698,57 @@ ${rows} } // Process CSV files - var refs = {}; - var errorCount = 0; - var errors = ''; - var indexOfURLs = []; - var checkedSourceHtmlScriptFiles = []; + let refs = {}; + let errorCount = 0; + let errors = ''; + let indexOfURLs = []; + let checkedSourceHtmlScriptFiles = []; + /** + * @param {string} id + * @param {string} error + */ function addTestError(id, error) { errorCount += 1; errors += '[Test ' + id + ']: ' + error + '\n'; } - function addCommandError({ testId, task }, key) { + /** + * @param {object} options + * @param {string} options.testId + * @param {object} options.target + * @param {string} key + */ + function addCommandError( + { + testId, + target: { + at: { key: atKey }, + }, + }, + key + ) { + errorCount += 1; + errors += `[Command]: The key reference "${key}" found in "${directory}/data/${atKey}-commands.csv" for "test id ${testId}" is invalid. Command may not be defined in "tests/commands.json".\n`; + } + + /** + * @param {object} options + * @param {string} options.testId + * @param {object} options.target + * @param {string} assertion + */ + function addCommandAssertionExceptionError( + { + testId, + target: { + at: { key: atKey }, + }, + }, + assertion + ) { errorCount += 1; - errors += `[Command]: The key reference "${key}" found in "${directory}/data/commands.csv" for "test id ${testId}: ${task}" is invalid. Command may not be defined in "tests/resources/keys.mjs".\n`; + errors += `[Command]: assertionExceptions reference "${assertion}" found in "${directory}/data/${atKey}-commands.csv" for "test id ${testId}" is invalid.\n`; } const newTestPlan = newBuild.find(`tests/${path.basename(testPlanBuildDirectory)}`); @@ -595,7 +773,14 @@ ${rows} // intended to be an internal helper to reduce some code duplication and make logging for csv errors simpler async function readCSVFile(filePath, rowValidator = identity => identity) { - const rawCSV = await readCSV(testPlanRecord.find(filePath)); + let rawCSV = []; + try { + rawCSV = await readCSV(testPlanRecord.find(filePath)); + } catch (e) { + log.warning(`WARNING: Error reading ${path.join(testPlanDirectory, filePath)}`); + return rawCSV; + } + let index = 0; function printError(message) { // line number is index+2 @@ -627,101 +812,73 @@ ${rows} return rawCSV; } - function validateReferencesKeys(row) { - if (typeof row.refId !== 'string' || typeof row.value !== 'string') { - throw new Error('Row missing refId or value'); - } - return row; - } + for (const row of referencesCsv) { + const { + references: { aria, htmlAam }, + } = supportJson; - const validCommandKeys = /^(?:testId|task|mode|at|command[A-Z])$/; - const numericKeyFormat = /^_(\d+)$/; - function validateCommandsKeys(row) { - // example header: - // testId,task,mode,at,commandA,commandB,commandC,commandD,commandE,commandF - for (const key of Object.keys(row)) { - if (numericKeyFormat.test(key)) { - throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); - } else if (!validCommandKeys.test(key)) { - throw new Error(`Unknown commands.csv key: ${key} - check header row?`); - } - } - if ( - !( - row.testId?.length && - row.task?.length && - row.mode?.length && - row.at?.length && - row.commandA?.length - ) - ) { - throw new Error('Missing one of required testId, task, mode, at, commandA'); - } - return row; - } + let refId = row.refId.trim(); + let type = row.type.trim(); + let value = row.value.trim(); + let linkText = row.linkText.trim(); - const validTestsKeys = - /^(?:testId|title|appliesTo|mode|task|setupScript|setupScriptDescription|refs|instructions|assertion(?:[1-9]|[1-2][0-9]|30))$/; - function validateTestsKeys(row) { - // example header: - // testId,title,appliesTo,mode,task,setupScript,setupScriptDescription,refs,instructions,assertion1,assertion2,assertion3,assertion4,assertion5,assertion6,assertion7 - for (const key of Object.keys(row)) { - if (numericKeyFormat.test(key)) { - throw new Error(`Column found without header row, ${+key.substring(1) + 1}`); - } else if (!validTestsKeys.test(key)) { - throw new Error(`Unknown tests.csv key: ${key} - check header row?`); - } - } - if ( - !( - row.testId?.length && - row.title?.length && - row.appliesTo?.length && - row.mode?.length && - row.task?.length - ) - ) { - throw new Error('Missing one of required testId, title, appliesTo, mode, task'); + if (type === 'aria') { + value = `${aria.baseUrl}${aria.fragmentIds[value]}`; + linkText = `${linkText} ${aria.linkText}`; } - return row; - } - const [refRows, atCommands, tests] = await Promise.all([ - readCSVFile('data/references.csv', validateReferencesKeys), - readCSVFile('data/commands.csv', validateCommandsKeys), - readCSVFile('data/tests.csv', validateTestsKeys), - ]); + if (type === 'htmlAam') { + value = `${htmlAam.baseUrl}${htmlAam.fragmentIds[value]}`; + linkText = `${linkText} ${htmlAam.linkText}`; + } - for (const row of refRows) { - refs[row.refId] = row.value.trim(); + refs[refId] = { + type, + value, + linkText, + }; } - const scripts = loadScripts(scriptsRecord); + const scriptsSource = loadScriptsSource(scriptsRecord); + const commandsParsed = parseCommandCSVRowV2( + { + commands: parsedAtCommandsCsvs, + }, + commandsJson + ); + const { + references: { aria, htmlAam }, + } = supportJson; + const referencesParsed = parseReferencesCSV(referencesCsv, { aria, htmlAam }); - const commandsParsed = atCommands.map(parseCommandCSVRow); - const testsParsed = tests.map(parseTestCSVRow); - const referencesParsed = parseReferencesCSV(refRows); + const keyDefs = flattenObject(commandsJson); const keysParsed = parseKeyMap(keyDefs); - const supportParsed = parseSupport(support); + const supportParsed = parseSupport(supportJson); + // TODO: This causes 'delete' to incorrectly not be recognized as a key, why? const keysValidated = validateKeyMap(keysParsed, { addKeyMapError(reason) { errorCount += 1; - errors += `[resources/keys.mjs]: ${reason}\n`; + errors += `[commands.json]: ${reason}\n`; }, }); const supportQueryables = { - at: Queryable.from('at', supportParsed.ats), - atGroup: Queryable.from('atGroup', supportParsed.atGroups), + ats: Queryable.from('ats', supportParsed.ats), + atGroups: Queryable.from('atGroups', supportParsed.atGroups), + references: Queryable.from('references', supportParsed.ats), + testPlanStrings: Queryable.from('testPlanStrings', supportParsed.ats), }; + const keyQueryable = Queryable.from('key', keysValidated); + const assertionQueryables = Queryable.from('assertions', assertionsCsv); const commandLookups = { key: keyQueryable, support: supportQueryables, + assertions: assertionQueryables, }; const commandsValidated = commandsParsed.map(command => - validateCommand(command, commandLookups, { addCommandError }) + validateCommand(command, commandLookups, { addCommandError, addCommandAssertionExceptionError }) ); const referenceQueryable = Queryable.from('reference', referencesParsed); @@ -740,8 +897,8 @@ ${rows} const testLookups = { command: Queryable.from('command', commandsValidated), mode: Queryable.from('mode', validModes), + script: Queryable.from('script', scriptsSource), reference: referenceQueryable, - script: Queryable.from('script', scripts), support: supportQueryables, }; const testsValidated = testsParsed.map(test => @@ -760,42 +917,120 @@ ${rows} reason => log.warning(`[${examplePathOriginal}]: ${reason.message}`), () => createExampleScriptsTemplate(exampleRecord) ); - const exampleScriptedFiles = [{ name: '', source: '' }, ...scripts].map(({ name, source }) => ({ - name, - path: examplePathTemplate(name), - content: exampleTemplate.render(exampleTemplateParams(name, source)).toString(), - })); + const exampleScriptedFiles = [{ name: '', source: '' }, ...scriptsSource].map( + ({ name, source }) => ({ + name, + path: examplePathTemplate(name), + content: exampleTemplate.render(exampleTemplateParams(name, source)).toString(), + }) + ); const exampleScriptedFilesQueryable = Queryable.from('example', exampleScriptedFiles); - const commandQueryable = Queryable.from('command', commandsValidated); const testsCollected = testsValidated.flatMap(test => { - return test.target.at.map(({ key }) => + return test.target.ats.map(({ key, settings }) => collectTestData({ test, command: commandQueryable.where({ testId: test.testId, - target: { at: { key } }, + target: { at: { key, settings } }, }), reference: referenceQueryable, + supportAt: supportQueryables.ats, example: exampleScriptedFilesQueryable, - key: keyQueryable, - modeInstructionTemplate: MODE_INSTRUCTION_TEMPLATES_QUERYABLE, + modeInstructionTemplate: MODE_INSTRUCTION_TEMPLATES_QUERYABLE(supportJson), }) ); }); + // So there can be a single {at}.collected.(json|html) instead of + // multiple {at}-{settings}.collected.(json|html) + let combinedCollectedTests = []; + const _testsCollected = JSON.parse(JSON.stringify(testsCollected.slice(0))); + + // { eg. jaws: [ ... ], nvda: [ ... ], vo: [ ... ] } + const collectedTestsGroupedByAt = {}; + allAtKeys.forEach(atKey => { + collectedTestsGroupedByAt[atKey] = _testsCollected.filter( + collected => collected.target.at.key === atKey + ); + }); + + for (const key in collectedTestsGroupedByAt) { + const collectedTestAtGroup = collectedTestsGroupedByAt[key]; + + // Get the unique testIds found for the at + const collectedTestsGroupedByAtTestIds = [ + ...new Set(collectedTestAtGroup.map(c => c.info.testId)), + ]; + + // Group by the testIds + for (const testId of collectedTestsGroupedByAtTestIds) { + const collectedTestsGroupedByAtGroupedByTestIds = collectedTestAtGroup.filter( + c => c.info.testId === testId + ); + + // Declare a common collectedTest that will be modified (if needed) + let common = collectedTestsGroupedByAtGroupedByTestIds[0]; + const settings = common.target.at.settings; + common = { + ...common, + instructions: { + ...common.instructions, + mode: { [settings]: common.instructions.mode }, + }, + commands: common.commands.map(command => ({ + ...command, + settings, + })), + }; + + // Iterate over the remaining collected tests + if (collectedTestsGroupedByAtGroupedByTestIds.length > 1) { + for (const additional of collectedTestsGroupedByAtGroupedByTestIds.slice(1)) { + const settings = additional.target.at.settings; + + common.target.at.settings = `${common.target.at.settings}_${settings}`; + common.instructions.mode[settings] = additional.instructions.mode; + common.commands.push( + ...additional.commands.map(command => ({ + ...command, + settings, + })) + ); + } + } + + combinedCollectedTests.push(common); + } + } + const files = [ - ...createScriptFiles(scripts, testPlanBuildDirectory), + ...createScriptFiles(scriptsSource, testPlanBuildDirectory), ...exampleScriptedFiles.map(({ path: pathSuffix, content }) => ({ path: path.join('build', 'tests', path.basename(directory), pathSuffix), content, })), - ...testsCollected.map(collectedTest => - createCollectedTestFile(collectedTest, testPlanBuildDirectory) - ), - ...testsCollected.map(collectedTest => - createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory) - ), + // TODO: If there is a need to individually view jaws-pcCursor or nvda-browseMode for example + // ...testsCollected.map(collectedTest => + // createCollectedTestFile(collectedTest, testPlanBuildDirectory) + // ), + // ...testsCollected.map(collectedTest => + // createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory) + // ), + ...combinedCollectedTests.map(collectedTest => { + const presentationNumber = parseInt(collectedTest.info.presentationNumber) + .toString() + .padStart(2, '0'); + const fileName = `test-${presentationNumber}-${collectedTest.info.testId}-${collectedTest.target.at.key}`; + return createCollectedTestFile(collectedTest, testPlanBuildDirectory, fileName); + }), + ...combinedCollectedTests.map(collectedTest => { + const presentationNumber = parseInt(collectedTest.info.presentationNumber) + .toString() + .padStart(2, '0'); + const fileName = `test-${presentationNumber}-${collectedTest.info.testId}-${collectedTest.target.at.key}`; + return createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory, fileName); + }), ]; files.forEach(file => { generateSourceHtmlScriptFile(file.path, file.content); @@ -819,16 +1054,18 @@ ${rows} }); } - const atCommandsMap = createCommandTuplesATModeTaskLookup(commandsValidated); + const atCommandsMap = + createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber(commandsValidated); + emitFile( path.join(testPlanBuildDirectory, 'commands.json'), beautify(atCommandsMap, null, 2, 40) ); log('Creating the following test files: '); - tests.forEach(function (test) { + testsParsed.forEach(function (testParsed, index) { try { - const [url, applies_to_at] = createTestFile(test, refs, atCommandsMap, { + const [url, applies_to_at] = createTestFile(testParsed, refs, atCommandsMap, { addTestError, emitFile, scriptsRecord, @@ -836,14 +1073,15 @@ ${rows} }); indexOfURLs.push({ - id: test.testId, - title: test.title, + seq: index + 1, + id: testParsed.testId, + title: testParsed.title, href: url, - script: test.setupScript, + script: testParsed.setupScript, applies_to_at: applies_to_at, }); - log('[Test ' + test.testId + ']: ' + url); + log('[Test ' + testParsed.testId + ']: ' + url); } catch (err) { log.warning(err); } @@ -870,7 +1108,7 @@ ${rows} ); log.warning(errors); } else { - log('No validation errors detected\n'); + log(`No validation errors detected for ${directory}\n`); } return { isSuccessfulRun: errorCount === 0, suppressedMessages }; @@ -917,7 +1155,7 @@ function readCSV(record) { * @param {FileRecordChain} testPlanJS * @returns {AriaATParsed.ScriptSource[]} */ -function loadScripts(testPlanJS) { +function loadScriptsSource(testPlanJS) { return testPlanJS.filter({ glob: ',*.js' }).entries.map(({ name: fileName, text: source }) => { const name = path.basename(fileName, '.js'); const modulePath = path.posix.join('scripts', `${name}.module.js`); @@ -990,25 +1228,45 @@ function createScriptFiles(scripts, testPlanBuildDirectory) { } /** - * @param {Object} keyLines + * @param {Object} keyDefs * @returns {AriaATParsed.KeyMap} */ function parseKeyMap(keyDefs) { const keyMap = {}; for (const id in keyDefs) { - keyMap[id] = { id, keystroke: keyDefs[id] }; + if (id.includes('.')) { + const [type, key] = id.split('.'); + keyMap[key] = { id: key, type, keystroke: keyDefs[id] }; + } } return keyMap; } /** * @param {AriaATCSV.Reference[]} referenceRows + * @param {AriaATCSV.SupportReference} aria + * @param {AriaATCSV.SupportReference} htmlAam * @returns {AriaATParsed.ReferenceMap} */ -function parseReferencesCSV(referenceRows) { +function parseReferencesCSV(referenceRows, { aria, htmlAam }) { const refMap = {}; - for (const { refId, value } of referenceRows) { - refMap[refId] = { refId, value: value.trim() }; + for (const { refId: _refId, type: _type, value: _value, linkText: _linkText } of referenceRows) { + let refId = _refId?.trim(); + let type = _type?.trim(); + let value = _value?.trim(); + let linkText = _linkText?.trim(); + + if (type === 'aria') { + value = `${aria.baseUrl}${aria.fragmentIds[value]}`; + linkText = `${linkText} ${aria.linkText}`; + } + + if (type === 'htmlAam') { + value = `${htmlAam.baseUrl}${htmlAam.fragmentIds[value]}`; + linkText = `${linkText} ${htmlAam.linkText}`; + } + + refMap[refId] = { refId, type, value, linkText }; } return refMap; } @@ -1019,34 +1277,62 @@ function parseReferencesCSV(referenceRows) { * @param {Queryable} data.key * @param {object} data.support * @param {Queryable<{key: string, name: string}>} data.support.at - * @param {object} [options] - * @param {function(AriaATParsed.Command, string): void} [options.addCommandError] + * @param {object} options + * @param {function(AriaATParsed.Command, string): void} options.addCommandError + * @param {function(AriaATParsed.Command, string): void} options.addCommandAssertionExceptionError * @returns {AriaATValidated.Command} */ -function validateCommand(commandParsed, data, { addCommandError = () => {} } = {}) { +function validateCommand( + commandParsed, + data, + { addCommandError = () => {}, addCommandAssertionExceptionError = () => {} } = {} +) { return { ...commandParsed, target: { ...commandParsed.target, at: { ...commandParsed.target.at, - ...mapDefined(data.support.at.where({ key: commandParsed.target.at.key }), ({ name }) => ({ + ...mapDefined(data.support.ats.where({ key: commandParsed.target.at.key }), ({ name }) => ({ name, })), }, }, - commands: commandParsed.commands.map(({ id, keypresses: commandKeypresses, ...rest }) => { - const keypresses = commandKeypresses.map(keypress => { - const key = data.key.where(keypress); - if (!key) { - addCommandError(commandParsed, keypress.id); + commands: commandParsed.commands.map(({ id, keypresses, assertionExceptions, ...rest }) => { + keypresses.forEach(keypress => { + if (keypress.id.includes('+')) { + const splitKeys = keypress.id.split('+'); + splitKeys.forEach(splitKey => { + const key = data.key.where({ id: splitKey }); + if (!key) { + addCommandError(commandParsed, keypress.id); + } + }); + } else { + const key = data.key.where({ id: keypress.id }); + if (!key) { + addCommandError(commandParsed, keypress.id); + } } - return key || {}; }); + return { - id: id, - keystroke: keypresses.map(({ keystroke }) => keystroke).join(', then '), + id, keypresses, + assertionExceptions: assertionExceptions.map(each => { + const [_priority, assertionId] = each.split(':'); + const priority = Number(_priority); + + if (isNaN(priority)) { + addCommandAssertionExceptionError(commandParsed, each); + } else { + if (!/^[0123]$/.test(_priority)) { + addCommandAssertionExceptionError(commandParsed, each); + } + } + + return { priority, assertionId }; + }), ...rest, }; }), @@ -1055,83 +1341,44 @@ function validateCommand(commandParsed, data, { addCommandError = () => {} } = { /** * @param {AriaATParsed.KeyMap} keyMap + * @param {object} options + * @param {function(string): void} options.addKeyMapError */ function validateKeyMap(keyMap, { addKeyMapError }) { - if (!keyMap.ALT_DELETE) { - addKeyMapError(`ALT_DELETE is not defined in keys module.`); - } - if (!keyMap.INS_Z) { - addKeyMapError(`INS_Z is not defined in keys module.`); - } - if (!keyMap.ESC) { - addKeyMapError(`ESC is not defined in keys module.`); - } - if (!keyMap.INS_SPACE) { - addKeyMapError(`INS_SPACE is not defined in keys module.`); - } - if (!keyMap.LEFT) { - addKeyMapError(`LEFT is not defined in keys module.`); - } - if (!keyMap.RIGHT) { - addKeyMapError(`RIGHT is not defined in keys module.`); - } + if (!keyMap.alt) addKeyMapError('"alt" is not defined in keys module.'); + if (!keyMap.del) addKeyMapError('"del" is not defined in keys module.'); + if (!keyMap.ins) addKeyMapError('"ins" is not defined in keys module.'); + if (!keyMap.z) addKeyMapError('"z" is not defined in keys module.'); + if (!keyMap.esc) addKeyMapError('"esc" is not defined in keys module.'); + if (!keyMap.space) addKeyMapError('"space" is not defined in keys module.'); + if (!keyMap.left) addKeyMapError('"left" is not defined in keys module.'); + if (!keyMap.right) addKeyMapError('"right" is not defined in keys module.'); + return keyMap; } -const MODE_INSTRUCTION_TEMPLATES_QUERYABLE = Queryable.from('modeInstructionTemplate', [ - { - at: 'jaws', - mode: 'reading', - render: data => { - const altDelete = data.key.where({ id: 'ALT_DELETE' }); - const esc = data.key.where({ id: 'ESC' }); - return `Verify the Virtual Cursor is active by pressing ${altDelete.keystroke}. If it is not, exit Forms Mode to activate the Virtual Cursor by pressing ${esc.keystroke}.`; - }, - }, - { - at: 'jaws', - mode: 'interaction', - render: data => { - const altDelete = data.key.where({ id: 'ALT_DELETE' }); - const insZ = data.key.where({ id: 'INS_Z' }); - return `Verify the PC Cursor is active by pressing ${altDelete.keystroke}. If it is not, turn off the Virtual Cursor by pressing ${insZ.keystroke}.`; - }, - }, - { - at: 'nvda', - mode: 'reading', - render: data => { - const esc = data.key.where({ id: 'ESC' }); - return `Ensure NVDA is in browse mode by pressing ${esc.keystroke}. Note: This command has no effect if NVDA is already in browse mode.`; - }, - }, - { - at: 'nvda', - mode: 'interaction', - render: data => { - const insSpace = data.key.where({ id: 'INS_SPACE' }); - return `If NVDA did not make the focus mode sound when the test page loaded, press ${insSpace.keystroke} to turn focus mode on.`; - }, - }, - { - at: 'voiceover_macos', - mode: 'reading', - render: data => { - const left = data.key.where({ id: 'LEFT' }); - const right = data.key.where({ id: 'RIGHT' }); - return `Toggle Quick Nav ON by pressing the ${left.keystroke} and ${right.keystroke} keys at the same time.`; - }, - }, - { - at: 'voiceover_macos', - mode: 'interaction', - render: data => { - const left = data.key.where({ id: 'LEFT' }); - const right = data.key.where({ id: 'RIGHT' }); - return `Toggle Quick Nav OFF by pressing the ${left.keystroke} and ${right.keystroke} keys at the same time.`; - }, - }, -]); +const MODE_INSTRUCTION_TEMPLATES_QUERYABLE = support => { + const collection = []; + support.ats.forEach(at => { + const modes = Object.keys(at.settings); + modes.forEach(mode => { + collection.push({ + at: at.key, + mode: mode, + render: [at.defaultConfigurationInstructionsHTML, ...at.settings[mode].instructions], + }); + }); + + // Accounting for modeless AT configurations + collection.push({ + at: at.key, + mode: 'defaultMode', + render: [at.defaultConfigurationInstructionsHTML], + }); + }); + + return Queryable.from('modeInstructionTemplate', collection); +}; /** * @param {T} maybeDefined @@ -1157,39 +1404,51 @@ function mapDefined(maybeDefined, goal) { * @param {Queryable} data.script * @param {object} data.support * @param {Queryable<{key: string, name: string}>} data.support.at - * @param {Queryable<{key: string, name: string}>} data.support.atGroup + * @param {Queryable<{key: string, name: string}>} data.support.atGroups * @param {object} [options] * @param {function(string): void} [options.addTestError] * @returns {AriaATValidated.Test} */ function validateTest(testParsed, data, { addTestError = () => {} } = {}) { - if (!data.command.where({ task: testParsed.task })) { - addTestError(`"${testParsed.task}" does not exist in commands.csv file.`); + const assertionStatementTokenPattern = /\{([^{}]+)}/g; + const replacedAssertionStatement = (assertionStatement, matches, replacements) => { + matches.forEach( + match => + (assertionStatement = assertionStatement.replaceAll( + match, + replacements[match.slice(1, -1)] + )) + ); + return assertionStatement; + }; + + if (!data.command.where({ testId: testParsed.testId })) { + addTestError(`"${testParsed.testId}" does not exist in *-commands.csv file.`); } - testParsed.target.at.forEach(at => { - if (!data.support.atGroup.where({ key: at.key })) { + testParsed.target.ats.forEach(at => { + if (!data.support.atGroups.where({ key: at.key })) { addTestError(`"${at.key}" is not valid value for "appliesTo" property.`); } if ( !data.command.where({ - task: testParsed.task, target: { - at: { key: at.key }, - mode: testParsed.target.mode, + at: { key: at.key, settings: at.settings }, }, }) ) { addTestError( - `command is missing for the combination of task: "${testParsed.task}", mode: "${testParsed.target.mode}", and AT: "${at.key}"` + `command is missing for the combination of task: "${testParsed.testId}", mode: "${at.settings}", and AT: "${at.key}"` ); } }); - if (!data.mode.where(testParsed.target.mode)) { - addTestError(`"${testParsed.target.mode}" is not valid value for "mode" property.`); - } + testParsed.target.ats.forEach(at => { + if (!data.mode.where(at.settings)) { + addTestError(`"${at.settings}" is not valid value for "settings" property.`); + } + }); const references = testParsed.references.filter(({ refId }) => { if (!data.reference.where({ refId })) { @@ -1199,25 +1458,55 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) { return true; }); - if (testParsed.setupScript && !data.script.where({ name: testParsed.setupScript.name })) { + if (testParsed.setupScript && !data.script.where({ name: testParsed.setupScript.script })) { addTestError( - `Setup script does not exist: "${testParsed.setupScript.name}" for task "${testParsed.task}"` + `Setup script does not exist: "${testParsed.setupScript.script}" for task "${testParsed.testId}"` ); } const assertions = testParsed.assertions.map(assertion => { + assertion.tokenizedAssertionStatements = {}; + + // There are assertion tokens to account for + if (assertion.assertionStatement.includes('|')) { + const [genericAssertionStatement, tokenizedAssertionStatement] = + assertion.assertionStatement.split('|'); + + // Set fallback just in case tokenized statement or properties do not exist + assertion.assertionStatement = genericAssertionStatement; + + if (tokenizedAssertionStatement) { + const matches = tokenizedAssertionStatement.match(assertionStatementTokenPattern); + + testParsed.target.ats.forEach(at => { + const { assertionTokens } = data.support.ats.where({ key: at.key }); + const tokensExist = + assertionTokens && matches.every(match => assertionTokens[match.slice(1, -1)]); + + if (tokensExist) { + assertion.tokenizedAssertionStatements[at.key] = replacedAssertionStatement( + tokenizedAssertionStatement, + matches, + assertionTokens + ); + } + }); + } + } + if ( typeof assertion.priority === 'string' || - (assertion.priority !== 1 && assertion.priority !== 2) + (assertion.priority !== 1 && assertion.priority !== 2 && assertion.priority !== 3) ) { addTestError( - `Level value must be 1 or 2, value found was "${assertion.priority}" for assertion "${assertion.expectation}" (NOTE: level 2 defined for this assertion).` + `Level value must be 1, 2 or 3, value found was "${assertion.priority}" for assertion "${assertion.assertionStatement}" (NOTE: Priority 3 defined for this assertion).` ); return { - priority: 2, - expectation: assertion.expectation, + ...assertion, + priority: 3, }; } + return assertion; }); @@ -1231,19 +1520,18 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) { ...data.reference.where({ refId: ref.refId }), })), target: { - at: testParsed.target.at.map(at => ({ + ats: testParsed.target.ats.map(at => ({ ...at, - ...mapDefined(data.support.at.where({ key: at.key }), ({ name }) => ({ + ...mapDefined(data.support.ats.where({ key: at.key }), ({ name }) => ({ name, })), })), - mode: testParsed.target.mode, }, setupScript: - testParsed.setupScript && data.script.where({ name: testParsed.setupScript.name }) + testParsed.setupScript && data.script.where({ name: testParsed.setupScript.script }) ? { ...testParsed.setupScript, - ...data.script.where({ name: testParsed.setupScript.name }), + ...data.script.where({ name: testParsed.setupScript.script }), } : undefined, assertions, @@ -1254,33 +1542,31 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) { * @param {object} data * @param {AriaATValidated.Test} data.test * @param {AriaATValidated.Command} data.command - * @param {Queryable} data.key + * @param {Queryable<{screenText: string, instructions: [string]}>} data.supportAt * @param {Queryable<{name: string, path: string}>} data.example * @param {Queryable<{at: string, mode: string, render: function({key: *}): string}>} data.modeInstructionTemplate * @returns {AriaATFile.CollectedTest} */ -function collectTestData({ test, command, key, example, modeInstructionTemplate }) { +function collectTestData({ test, command, supportAt, example, modeInstructionTemplate }) { return { info: { testId: test.testId, - task: test.task, title: test.title, references: test.references, + presentationNumber: test.presentationNumber, }, target: { ...test.target, - at: command.target.at, + at: { ...command.target.at, raw: supportAt.where({ key: command.target.at.key }) }, referencePage: example.where({ name: test.setupScript ? test.setupScript.name : '' }).path, setupScript: test.setupScript, }, instructions: { - ...test.instructions, - mode: modeInstructionTemplate - .where({ - at: command.target.at.key, - mode: command.target.mode, - }) - .render({ key }), + instructions: test.instructions, + mode: modeInstructionTemplate.where({ + at: command.target.at.key, + mode: command.target.at.settings, + }).render, }, commands: command.commands, assertions: test.assertions, @@ -1296,16 +1582,18 @@ function encodeText(text) { /** * @param {AriaATFile.CollectedTest} test + * @param {string} testPlanBuildDirectory + * @param {string} fileName + * @returns {{path: string, content: Uint8Array}} */ -function createCollectedTestFile(test, testPlanBuildDirectory) { +function createCollectedTestFile(test, testPlanBuildDirectory, fileName = null) { + const testPresentationNumber = parseInt(test.info.presentationNumber).toString().padStart(2, '0'); + const testFileName = + fileName || + `test-${testPresentationNumber}-${test.info.testId}-${test.target.at.key}-${test.target.at.settings}`; + return { - path: path.join( - testPlanBuildDirectory, - `test-${test.info.testId.toString().padStart(2, '0')}-${test.info.task.replace( - /\s+/g, - '-' - )}-${test.target.mode}-${test.target.at.key}.collected.json` - ), + path: path.join(testPlanBuildDirectory, `${testFileName}.collected.json`), content: encodeText(beautify(test, null, 2, 40)), }; } @@ -1313,23 +1601,18 @@ function createCollectedTestFile(test, testPlanBuildDirectory) { /** * @param {AriaATFile.CollectedTest} test * @param {string} testPlanBuildDirectory + * @param {string} fileName * @returns {{path: string, content: Uint8Array}} */ -function createCollectedTestHtmlFile(test, testPlanBuildDirectory) { - const testJsonFileName = `test-${test.info.testId - .toString() - .padStart(2, '0')}-${test.info.task.replace(/\s+/g, '-')}-${test.target.mode}-${ - test.target.at.key - }.collected.json`; +function createCollectedTestHtmlFile(test, testPlanBuildDirectory, fileName = null) { + const testPresentationNumber = parseInt(test.info.presentationNumber).toString().padStart(2, '0'); + const testFileName = + fileName || + `test-${testPresentationNumber}-${test.info.testId}-${test.target.at.key}-${test.target.at.settings}`; + return { - path: path.join( - testPlanBuildDirectory, - `test-${test.info.testId.toString().padStart(2, '0')}-${test.info.task.replace( - /\s+/g, - '-' - )}-${test.target.mode}-${test.target.at.key}.collected.html` - ), - content: encodeText(renderCollectedTestHtml(test, testJsonFileName)), + path: path.join(testPlanBuildDirectory, `${testFileName}.collected.html`), + content: encodeText(renderCollectedTestHtml(test, `${testFileName}.collected.json`)), }; } diff --git a/package.json b/package.json index 3f41e5e66..95d0dd012 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "count": "node ./scripts/count-assertions.js", "lint": "eslint .", "build": "npm run create-all-tests && npm run review-tests", - "validate": "npm run create-all-tests -- --validate", + "build-v1": "npm run create-all-tests -- --version1 --verbose && npm run review-tests", + "build-v2": "npm run create-all-tests -- --version2 --verbose && npm run review-tests", + "validate": "npm run create-all-tests -- --verbose --validate", "cleanup": "rimraf build", "create-all-tests": "cross-var node scripts/create-all-tests.js --testplan=$npm_config_testplan", "review-tests": "cross-var node --experimental-modules scripts/test-reviewer.mjs --testplan=$npm_config_testplan", diff --git a/scripts/create-all-tests.js b/scripts/create-all-tests.js index aa4d45305..c58266148 100644 --- a/scripts/create-all-tests.js +++ b/scripts/create-all-tests.js @@ -2,7 +2,12 @@ const path = require('path'); const fse = require('fs-extra'); -const { processTestDirectory } = require('../lib/data/process-test-directory'); +const { + processTestDirectory: processTestDirectoryV2, +} = require('../lib/data/process-test-directory'); +const { + processTestDirectory: processTestDirectoryV1, +} = require('../lib/data/process-test-directory-v1'); const args = require('minimist')(process.argv.slice(2), { alias: { @@ -10,6 +15,8 @@ const args = require('minimist')(process.argv.slice(2), { t: 'testplan', v: 'verbose', V: 'validate', + v1: 'version1', + v2: 'version2', }, }); @@ -26,6 +33,10 @@ if (args.help) { Generate tests and view a detailed report summary. -V, --validate Determine whether current test plans are valid (no errors present). + -v1, --version1 + Build the tests with the v1 format of the tests + -v2, --version2 + Build the tests with the v2 format of the tests `); process.exit(); } @@ -36,8 +47,11 @@ async function main() { // on some OSes, it seems the `npm_config_testplan` environment variable will come back as the actual variable name rather than empty if it does not exist const TARGET_TEST_PLAN = args.testplan && !args.testplan.includes('npm_config_testplan') ? args.testplan : null; // individual test plan to generate test assets for + const VERBOSE_CHECK = !!args.verbose; const VALIDATE_CHECK = !!args.validate; + const V1_CHECK = !!args.v1; + const V2_CHECK = !!args.v2; const scriptsDirectory = path.dirname(__filename); const rootDirectory = path.join(scriptsDirectory, '..'); @@ -58,15 +72,44 @@ async function main() { } const filteredTests = await Promise.all( - filteredTestPlans.map(directory => - processTestDirectory({ - directory: path.join('tests', directory), - args, - }).catch(error => { - error.directory = directory; - throw error; - }) - ) + filteredTestPlans.map(directory => { + let FALLBACK_V1_CHECK = false; + let FALLBACK_V2_CHECK = false; + + // Check if files exist for doing v2 build by default first, then try v1 + if (!V1_CHECK && !V2_CHECK) { + // Use existence of assertions.csv to determine if v2 format files exist for now + const assertionsCsvPath = path.join( + __dirname, + '../', + 'tests', + directory, + 'data', + 'assertions.csv' + ); + + if (fse.existsSync(assertionsCsvPath)) FALLBACK_V2_CHECK = true; + else FALLBACK_V1_CHECK = true; + } + + if (FALLBACK_V2_CHECK || V2_CHECK) { + return processTestDirectoryV2({ + directory: path.join('tests', directory), + args, + }).catch(error => { + error.directory = directory; + throw error; + }); + } else if (FALLBACK_V1_CHECK || V1_CHECK) { + return processTestDirectoryV1({ + directory: path.join('tests', directory), + args, + }).catch(error => { + error.directory = directory; + throw error; + }); + } + }) ).catch(error => { console.error(`ERROR: Unhandled exception thrown while processing "${error.directory}".`); if (!VERBOSE_CHECK) { diff --git a/scripts/review-index-template.mustache b/scripts/review-index-template.mustache index ab76db06f..ef19e13db 100644 --- a/scripts/review-index-template.mustache +++ b/scripts/review-index-template.mustache @@ -38,6 +38,7 @@
Sequence Task ID Testing Task
+ @@ -46,9 +47,10 @@ {{#patterns}} - - - + + + + diff --git a/scripts/review-template.mustache b/scripts/review-template.mustache index fdc248967..74063f23b 100644 --- a/scripts/review-template.mustache +++ b/scripts/review-template.mustache @@ -2,7 +2,7 @@ - Test plan review for pattern: {{pattern}} + {{title}} Test Plan | For Pattern: {{pattern}}
Title Pattern Index Page Review Page
{{name}}IndexReview{{title}}{{patternName}}IndexReview

{{numberOfTests}}

{{commitDescription}}