diff --git a/lib/data/command-tuples-at-mode-task-lookup.js b/lib/data/command-tuples-at-mode-task-lookup.js
index b8a54f2dc..625dcb7f6 100644
--- a/lib/data/command-tuples-at-mode-task-lookup.js
+++ b/lib/data/command-tuples-at-mode-task-lookup.js
@@ -31,4 +31,28 @@ function createCommandTuplesATModeTaskLookup(commands) {
return data;
}
-exports.createCommandTuplesATModeTaskLookup = createCommandTuplesATModeTaskLookup;
+function createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber(commands) {
+ return commands.reduce((carry, command) => {
+ const commandTask = carry[command.testId] || {};
+ const commandTaskMode = commandTask[command.target.at.settings || 'defaultMode'] || {};
+ const commandTaskModeAT = commandTaskMode[command.target.at.key] || [];
+ const commandTuples = command.commands.map(({ id, presentationNumber }) => [
+ `${id}|${presentationNumber}`,
+ ]);
+ return {
+ ...carry,
+ [command.testId]: {
+ ...commandTask,
+ [command.target.at.settings || 'defaultMode']: {
+ ...commandTaskMode,
+ [command.target.at.key]: [...commandTaskModeAT, ...commandTuples],
+ },
+ },
+ };
+ }, {});
+}
+
+module.exports = {
+ createCommandTuplesATModeTaskLookup,
+ createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber,
+};
diff --git a/lib/data/parse-command-csv-row.js b/lib/data/parse-command-csv-row.js
index 6cc5c1e2c..0265cfa65 100644
--- a/lib/data/parse-command-csv-row.js
+++ b/lib/data/parse-command-csv-row.js
@@ -67,4 +67,194 @@ function parseCommandCSVRow(commandRow) {
};
}
-exports.parseCommandCSVRow = parseCommandCSVRow;
+function flattenObject(obj, parentKey = '') {
+ const flattened = {};
+
+ for (const key in obj) {
+ if (typeof obj[key] === 'object') {
+ const subObject = flattenObject(obj[key], parentKey + key + '.');
+ Object.assign(flattened, subObject);
+ } else {
+ flattened[parentKey + key] = obj[key];
+ }
+ }
+
+ return flattened;
+}
+
+function sanitizeWhitespace(value) {
+ return value.replace(/\s+/g, ' ').trim();
+}
+
+function sanitizeCommand(command) {
+ return {
+ ...command,
+ command: sanitizeWhitespace(command.command),
+ };
+}
+
+function findValueByKey(keyMappings, keyToFindText) {
+ const keyToFind = keyToFindText.replace(/\s+/g, ' ').trim();
+ const keyMap = Object.keys(keyMappings);
+
+ // Need to specially handle VO modifier key combination
+ if (keyToFind === 'vo')
+ return findValuesByKeys(keyMappings, [keyMappings['modifierAliases.vo']])[0];
+
+ if (keyToFind.includes('modifiers.') || keyToFind.includes('keys.')) {
+ const parts = keyToFind.split('.');
+ const keyToCheck = parts[parts.length - 1]; // value after the '.'
+
+ if (keyMappings[keyToFind])
+ return {
+ value: keyMappings[keyToFind],
+ key: keyToCheck,
+ };
+
+ return null;
+ }
+
+ for (const key of keyMap) {
+ const parts = key.split('.');
+ const parentKey = parts[0];
+ const keyToCheck = parts[parts.length - 1]; // value after the '.'
+
+ if (keyToCheck === keyToFind) {
+ if (parentKey === 'modifierAliases') {
+ return findValueByKey(keyMappings, `modifiers.${keyMappings[key]}`);
+ } else if (parentKey === 'keyAliases') {
+ return findValueByKey(keyMappings, `keys.${keyMappings[key]}`);
+ }
+
+ return {
+ value: keyMappings[key],
+ key: keyToCheck,
+ };
+ }
+ }
+
+ // Return null if the key is not found
+ return null;
+}
+
+function findValuesByKeys(commandsMapping, keysToFind = []) {
+ const result = [];
+
+ const patternSepWithReplacement = (keyToFind, pattern, replacement) => {
+ if (keyToFind.includes(pattern)) {
+ let value = '';
+ let validKeys = true;
+ const keys = keyToFind.split(pattern);
+
+ for (const key of keys) {
+ const keyResult = findValueByKey(commandsMapping, key);
+ if (keyResult) value = value ? `${value}${replacement}${keyResult.value}` : keyResult.value;
+ else validKeys = false;
+ }
+ if (validKeys) return { value, key: keyToFind };
+ }
+
+ return null;
+ };
+
+ const patternSepHandler = keyToFind => {
+ let value = '';
+
+ if (keyToFind.includes(' ') && keyToFind.includes('+')) {
+ const keys = keyToFind.split(' ');
+ for (let [index, key] of keys.entries()) {
+ const keyToFindResult = findValueByKey(commandsMapping, key);
+ if (keyToFindResult) keys[index] = keyToFindResult.value;
+ if (key.includes('+')) keys[index] = patternSepWithReplacement(key, '+', '+').value;
+ }
+ value = keys.join(' then ');
+
+ return { value, key: keyToFind };
+ } else if (keyToFind.includes(' ')) return patternSepWithReplacement(keyToFind, ' ', ' then ');
+ else if (keyToFind.includes('+')) return patternSepWithReplacement(keyToFind, '+', '+');
+ };
+
+ for (const keyToFind of keysToFind) {
+ if (keyToFind.includes(' ') || keyToFind.includes('+')) {
+ result.push(patternSepHandler(keyToFind));
+ } else {
+ const keyToFindResult = findValueByKey(commandsMapping, keyToFind);
+ if (keyToFindResult) result.push(keyToFindResult);
+ }
+ }
+
+ return result;
+}
+
+function parseCommandCSVRowV2({ commands }, commandsJson) {
+ const commandsParsed = [];
+ const flattenedCommandsJson = flattenObject(commandsJson);
+
+ function createParsedCommand(command, atTargetInfo) {
+ const commandIds = command.command.split(' ');
+ const commandKVs = findValuesByKeys(flattenedCommandsJson, [command.command]);
+
+ const assertionExceptions = command.assertionExceptions.length
+ ? sanitizeWhitespace(command.assertionExceptions).split(' ')
+ : [];
+
+ const commands = commandKVs.map(e => ({
+ id: e.key,
+ keystroke: e.value,
+ keypresses: e.value.split(' then ').map((e, index) => ({
+ id: commandIds[index],
+ keystroke: e,
+ })),
+ presentationNumber: Number(command.presentationNumber),
+ assertionExceptions,
+ }));
+
+ // Also account for 'modeless' AT configurations with 'defaultMode'
+ const settingsValue = command.settings || 'defaultMode';
+
+ const foundCommandIndex = commandsParsed.findIndex(
+ e =>
+ e.testId === command.testId &&
+ e.target.at.key === atTargetInfo.key &&
+ e.target.at.settings === settingsValue &&
+ (e.commands.length
+ ? e.commands.every(
+ ({ presentationNumber }) =>
+ parseInt(presentationNumber) === parseInt(command.presentationNumber)
+ )
+ : true)
+ );
+
+ // Add new parsed command since none exist
+ if (foundCommandIndex < 0) {
+ commandsParsed.push({
+ testId: command.testId,
+ target: {
+ at: {
+ ...atTargetInfo,
+ settings: settingsValue,
+ },
+ },
+ commands,
+ });
+ } else {
+ commandsParsed[foundCommandIndex].commands.push(...commands);
+ }
+ }
+
+ const generateCommandsParsed = (commands, key) => {
+ for (const command of commands) createParsedCommand(command, { key });
+ };
+
+ commands.forEach(command =>
+ generateCommandsParsed(command.commands.map(sanitizeCommand), command.atKey)
+ );
+
+ return commandsParsed;
+}
+
+module.exports = {
+ parseCommandCSVRow,
+ parseCommandCSVRowV2,
+ flattenObject,
+};
diff --git a/lib/data/parse-support.js b/lib/data/parse-support.js
index 5aed27cd4..ed19102e3 100644
--- a/lib/data/parse-support.js
+++ b/lib/data/parse-support.js
@@ -22,6 +22,8 @@ function parseSupport(supportRaw) {
ats: [at],
})),
],
+ testPlanStrings: supportRaw.testPlanStrings,
+ references: supportRaw.references,
};
}
diff --git a/lib/data/parse-test-csv-row.js b/lib/data/parse-test-csv-row.js
index 66304e200..c0138e46e 100644
--- a/lib/data/parse-test-csv-row.js
+++ b/lib/data/parse-test-csv-row.js
@@ -84,4 +84,179 @@ function parseTestCSVRow(testRow) {
};
}
-exports.parseTestCSVRow = parseTestCSVRow;
+function isEmptyObject(object) {
+ return object && Object.keys(object).length === 0;
+}
+
+function parseTestCSVRowV2({ tests, assertions, scripts, commands }) {
+ const testsParsed = [];
+
+ // Create containers for all potentially found presentationNumbers; empty ones will be filtered out at the end.
+ // For eg. in the case where an {at-key}-commands.csv is missing.
+ // This also maintains the order for testRow.presentationNumber
+ for (let i = 0; i < tests.length; i++) {
+ testsParsed.push({});
+ }
+
+ function createParsedTest(arrPosition, test, command, atTargetInfo) {
+ // Create setupScript object
+ const setupScript = test.setupScript
+ ? scripts.find(script => test.setupScript === script.setupScript) // { setupScript, setupScriptDescription }
+ : undefined;
+
+ // Create assertions value
+ const assertionsValue = test.assertions
+ ? test.assertions.split(' ').map(assertion => {
+ // TODO: Return error if foundAssertion undefined
+ const foundAssertion = assertions.find(e => e.assertionId === assertion);
+ return foundAssertion
+ ? { ...foundAssertion, priority: Number(foundAssertion.priority) }
+ : {};
+ })
+ : undefined;
+
+ // Create references values
+ let referencesValue = assertionsValue
+ ? assertionsValue.flatMap(assertion => assertion.refIds.trim().split(' '))
+ : [];
+ referencesValue = referencesValue.filter(e => e !== '');
+
+ if (referencesValue.length) {
+ // Remove duplicates
+ referencesValue = [...new Set(referencesValue)];
+
+ // Format to [ { refId: '' } ]
+ referencesValue = referencesValue.map(refId => {
+ return { refId };
+ });
+ }
+
+ // Also account for 'modeless' AT configurations with 'defaultMode'
+ // TODO: Account for a list of settings as described by https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#settings
+ const settingsValue = command.settings || 'defaultMode';
+
+ if (isEmptyObject(testsParsed[arrPosition])) {
+ testsParsed[arrPosition] = {
+ testId: test.testId,
+ title: test.title,
+ references: referencesValue,
+ presentationNumber: Number(test.presentationNumber),
+ target: {
+ ats: [
+ {
+ ...atTargetInfo,
+ settings: settingsValue,
+ },
+ ],
+ },
+ setupScript: setupScript
+ ? {
+ script: setupScript.setupScript,
+ scriptDescription: setupScript.setupScriptDescription,
+ }
+ : setupScript,
+ instructions: test.instructions,
+ assertions: assertionsValue,
+ };
+ } else {
+ if (
+ !testsParsed[arrPosition].target.ats.find(
+ e => e.key === atTargetInfo.key && e.settings === settingsValue
+ )
+ )
+ testsParsed[arrPosition].target.ats.push({
+ ...atTargetInfo,
+ settings: settingsValue,
+ });
+ }
+ }
+
+ const generateTestsParsed = (commands, key) => {
+ for (const command of commands) {
+ const findTest = test => test.testId === command.testId;
+
+ const test = tests.find(findTest);
+ const testIndex = tests.findIndex(findTest);
+ createParsedTest(testIndex, test, command, { key });
+ }
+ };
+
+ const attachCommandInfo = (commands, key) => {
+ function findCommandInfo({ command, presentationNumber, settings, testId }, commandInfo) {
+ return (
+ testId === commandInfo.testId &&
+ command === commandInfo.command &&
+ settings === commandInfo.settings &&
+ presentationNumber === commandInfo.presentationNumber
+ );
+ }
+
+ for (const command of commands) {
+ testsParsed.forEach(test => {
+ test.assertions?.forEach(assertion => {
+ if (
+ command.testId === test.testId &&
+ test.target.ats.some(
+ at => at.key === key && at.settings === (command.settings || 'defaultMode')
+ )
+ ) {
+ if (!assertion.commandInfo) assertion.commandInfo = {};
+ if (!assertion.commandInfo[key]) assertion.commandInfo[key] = [];
+
+ const commandInfo = {
+ testId: command.testId,
+ command: command.command,
+ settings: command.settings || 'defaultMode',
+ presentationNumber: Number(command.presentationNumber),
+ };
+
+ // assertion level commandInfo, useful for getting assertionExceptions
+ const assertionCommandInfoExists = assertion.commandInfo[key].find(each =>
+ findCommandInfo(each, commandInfo)
+ );
+ if (!assertionCommandInfoExists) {
+ const assertionCommandInfo = {
+ ...commandInfo,
+ assertionExceptions: command.assertionExceptions,
+ };
+ assertion.commandInfo[key].push(assertionCommandInfo);
+ }
+
+ // Test level commandInfo, useful for getting review level information
+ if (!test.commandsInfo) test.commandsInfo = {};
+ if (!test.commandsInfo[key]) test.commandsInfo[key] = [];
+ const testCommandInfoExists = test.commandsInfo[key].find(each =>
+ findCommandInfo(each, commandInfo)
+ );
+
+ if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo);
+ }
+ });
+ });
+ }
+ };
+
+ // Maintain the order of commands by commandRow.presentationNumber
+ const commandsSort = (a, b) =>
+ parseFloat(a.presentationNumber) - parseFloat(b.presentationNumber);
+
+ /*const GENERATE_TESTS = [
+ [jawsCommands, 'jaws'],
+ [nvdaCommands, 'nvda'],
+ [voCommands, 'voiceover_macos'],
+ ];*/
+ const GENERATE_TESTS = commands.map(command => [command.commands, command.atKey]);
+ GENERATE_TESTS.forEach(([commands, key]) => {
+ if (commands.length) generateTestsParsed(commands.sort(commandsSort), key);
+ });
+ GENERATE_TESTS.forEach(([commands, key]) => {
+ if (commands.length) attachCommandInfo(commands.sort(commandsSort), key);
+ });
+
+ return testsParsed.filter(testParsed => Object.keys(testParsed).length);
+}
+
+module.exports = {
+ parseTestCSVRow,
+ parseTestCSVRowV2,
+};
diff --git a/lib/data/process-test-directory-v1.js b/lib/data/process-test-directory-v1.js
new file mode 100644
index 000000000..c286e2c78
--- /dev/null
+++ b/lib/data/process-test-directory-v1.js
@@ -0,0 +1,1386 @@
+///
+///
+///
+///
+///
+
+'use strict';
+
+const path = require('path');
+const { Readable } = require('stream');
+const {
+ types: { isArrayBufferView, isArrayBuffer },
+} = require('util');
+
+const csv = require('csv-parser');
+const fse = require('fs-extra');
+const beautify = require('json-beautify');
+
+const { validate } = require('../util/error');
+const { reindent } = require('../util/lines');
+const { Queryable } = require('../util/queryable');
+const { FileRecordChain } = require('../util/file-record-chain');
+
+const { parseSupport } = require('./parse-support');
+const { parseTestCSVRow } = require('./parse-test-csv-row');
+const { parseCommandCSVRow } = require('./parse-command-csv-row');
+const { createCommandTuplesATModeTaskLookup } = require('./command-tuples-at-mode-task-lookup');
+
+const { renderHTML: renderCollectedTestHtml } = require('./templates/collected-test.html');
+const { createExampleScriptsTemplate } = require('./example-scripts-template');
+
+/**
+ * @param {string} directory - path to directory of data to be used to generate test
+ * @param {object} args={}
+ */
+const processTestDirectory = async ({ directory, args = {} }) => {
+ let VERBOSE_CHECK = false;
+ let VALIDATE_CHECK = false;
+
+ let suppressedMessages = 0;
+
+ /**
+ * @param {string} message - message to be logged
+ * @param {object} [options]
+ * @param {boolean} [options.severe=false] - indicates whether the message should be viewed as an error or not
+ * @param {boolean} [options.force=false] - indicates whether this message should be forced to be outputted regardless of verbosity level
+ */
+ const log = (message, { severe = false, force = false } = {}) => {
+ if (VERBOSE_CHECK || force) {
+ if (severe) console.error(message);
+ else console.log(message);
+ } else {
+ // Output no logs
+ suppressedMessages += 1; // counter to indicate how many messages were hidden
+ }
+ };
+
+ /**
+ * @param {string} message - error message
+ */
+ log.warning = message => log(message, { severe: true, force: true });
+
+ /**
+ * Log error then exit the process.
+ * @param {string} message - error message
+ */
+ log.error = message => {
+ log.warning(message);
+ process.exit(1);
+ };
+
+ // setup from arguments passed to npm script
+ VERBOSE_CHECK = !!args.verbose;
+ VALIDATE_CHECK = !!args.validate;
+
+ const validModes = ['reading', 'interaction', 'item'];
+
+ // cwd; @param rootDirectory is dependent on this file not moving from the `lib/data` folder
+ const libDataDirectory = path.dirname(__filename);
+ const rootDirectory = path.join(libDataDirectory, '../..');
+
+ const testsDirectory = path.join(rootDirectory, 'tests');
+ const testPlanDirectory = path.join(rootDirectory, directory);
+
+ const resourcesDirectory = path.join(testsDirectory, 'resources');
+
+ const supportFilePath = path.join(testsDirectory, 'support.json');
+ const atCommandsCsvFilePath = path.join(testPlanDirectory, 'data', 'commands.csv');
+ const testsCsvFilePath = path.join(testPlanDirectory, 'data', 'testsV1.csv');
+ const referencesCsvFilePath = path.join(testPlanDirectory, 'data', 'referencesV1.csv');
+
+ // build output folders and file paths setup
+ const buildDirectory = path.join(rootDirectory, 'build');
+ const testPlanBuildDirectory = path.join(buildDirectory, directory);
+ const indexFileBuildOutputPath = path.join(testPlanBuildDirectory, 'index.html');
+
+ let backupTestsCsvFile, backupReferencesCsvFile;
+
+ // create build directory if it doesn't exist
+ fse.mkdirSync(buildDirectory, { recursive: true });
+
+ const existingBuildPromise = FileRecordChain.read(buildDirectory, {
+ glob: [
+ '',
+ 'tests',
+ `tests/${path.basename(directory)}`,
+ `tests/${path.basename(directory)}/**`,
+ 'tests/resources',
+ 'tests/resources/*',
+ 'tests/support.json',
+ ].join(','),
+ });
+
+ const [testPlanRecord, resourcesOriginalRecord, supportRecord] = await Promise.all(
+ [testPlanDirectory, resourcesDirectory, supportFilePath].map(filepath =>
+ FileRecordChain.read(filepath)
+ )
+ );
+
+ const scriptsRecord = testPlanRecord.find('data/js');
+ const resourcesRecord = resourcesOriginalRecord.filter({ glob: '{aria-at-*,keys,vrender}.mjs' });
+
+ const newBuild = new FileRecordChain({
+ entries: [
+ {
+ name: 'tests',
+ entries: [
+ {
+ name: path.basename(directory),
+ entries: testPlanRecord.filter({ glob: 'reference{,/**}' }).record.entries,
+ },
+ { name: 'resources', ...resourcesRecord.record },
+ { name: 'support.json', ...supportRecord.record },
+ ],
+ },
+ ],
+ });
+
+ const keyDefs = {};
+ const supportJson = JSON.parse(supportRecord.text);
+
+ let allATKeys = supportJson.ats.map(({ key }) => key);
+ let allATNames = supportJson.ats.map(({ name }) => name);
+
+ const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys);
+
+ if (!testPlanRecord.isDirectory()) {
+ log.error(`The test directory '${testPlanDirectory}' does not exist. Check the path to tests.`);
+ }
+
+ if (!testPlanRecord.find('data/commands.csv').isFile()) {
+ log.error(
+ `The at-commands.csv file does not exist. Please create '${atCommandsCsvFilePath}' file.`
+ );
+ }
+
+ if (!testPlanRecord.find('data/testsV1.csv').isFile()) {
+ // Check if original file can be processed
+ if (!testPlanRecord.find('data/tests.csv').isFile()) {
+ log.error(`The testsV1.csv file does not exist. Please create '${testsCsvFilePath}' file.`);
+ } else {
+ backupTestsCsvFile = 'data/tests.csv';
+ }
+ }
+
+ if (!testPlanRecord.find('data/referencesV1.csv').isFile()) {
+ // Check if original file can be processed
+ if (!testPlanRecord.find('data/references.csv').isFile()) {
+ log.error(
+ `The referencesV1.csv file does not exist. Please create '${referencesCsvFilePath}' file.`
+ );
+ } else {
+ backupReferencesCsvFile = 'data/references.csv';
+ }
+ }
+
+ // get Keys that are defined
+ try {
+ // read contents of the file
+ const keys = resourcesRecord.find('keys.mjs').text;
+
+ // split the contents by new line
+ const lines = keys.split(/\r?\n/);
+
+ // print all lines
+ lines.forEach(line => {
+ let parts1 = line.split(' ');
+ let parts2 = line.split('"');
+
+ if (parts1.length > 3) {
+ let code = parts1[2].trim();
+ keyDefs[code] = parts2[1].trim();
+ }
+ });
+ } catch (err) {
+ log.warning(err);
+ }
+
+ function cleanTask(task) {
+ return task.replace(/'/g, '').replace(/;/g, '').trim().toLowerCase();
+ }
+
+ /**
+ * Create Test File
+ * @param {AriaATCSV.Test} test
+ * @param refs
+ * @param commands
+ * @returns {(string|*[])[]}
+ */
+ function createTestFile(
+ test,
+ refs,
+ commands,
+ { addTestError, emitFile, scriptsRecord, exampleScriptedFilesQueryable }
+ ) {
+ let scripts = [];
+
+ // default setupScript if test has undefined setupScript
+ if (!scriptsRecord.find(`${test.setupScript}.js`).isFile()) test.setupScript = '';
+
+ function getModeValue(value) {
+ let v = value.trim().toLowerCase();
+ if (!validModes.includes(v)) {
+ addTestError(test.testId, '"' + value + '" is not valid value for "mode" property.');
+ }
+ return v;
+ }
+
+ function getTask(t) {
+ let task = cleanTask(t);
+
+ if (typeof commands[task] !== 'object') {
+ addTestError(test.testId, '"' + task + '" does not exist in commands.csv file.');
+ }
+
+ return task;
+ }
+
+ function getAppliesToValues(values) {
+ function checkValue(value) {
+ let v1 = value.trim().toLowerCase();
+ for (let i = 0; i < validAppliesTo.length; i++) {
+ let v2 = validAppliesTo[i];
+ if (v1 === v2.toLowerCase()) {
+ return v2;
+ }
+ }
+ return false;
+ }
+
+ // check for individual assistive technologies
+ let items = values.split(',');
+ let newValues = [];
+ items.filter(item => {
+ let value = checkValue(item);
+ if (!value) {
+ addTestError(test.testId, '"' + item + '" is not valid value for "appliesTo" property.');
+ }
+
+ newValues.push(value);
+ });
+
+ return newValues;
+ }
+
+ /**
+ * Determines priority level (default is 1) of assertion string, then adds it to the collection of assertions for
+ * the test plan
+ * @param {string} a - Assertion string to be evaluated
+ */
+ function addAssertion(a) {
+ let level = '1';
+ let str = a;
+ a = a.trim();
+
+ // matches a 'colon' when preceded by either of the digits 1 OR 2 (SINGLE CHARACTER), at the start of the string
+ let parts = a.split(/(?<=^[1-2]):/g);
+
+ if (parts.length === 2) {
+ level = parts[0];
+ str = parts[1].substring(0);
+ if (level !== '1' && level !== '2') {
+ addTestError(
+ test.testId,
+ "Level value must be 1 or 2, value found was '" +
+ level +
+ "' for assertion '" +
+ str +
+ "' (NOTE: level 2 defined for this assertion)."
+ );
+ level = '2';
+ }
+ }
+
+ if (a.length) {
+ assertions.push([level, str]);
+ }
+ }
+
+ function getReferences(example, testRefs) {
+ let links = '';
+
+ if (typeof example === 'string' && example.length) {
+ links += ` \n`;
+ }
+
+ let items = test.refs.split(' ');
+ items.forEach(function (item) {
+ item = item.trim();
+
+ if (item.length) {
+ if (typeof refs[item] === 'string') {
+ links += ` \n`;
+ } else {
+ addTestError(test.testId, 'Reference does not exist: ' + item);
+ }
+ }
+ });
+
+ return links;
+ }
+
+ function addSetupScript(scriptName) {
+ let script = '';
+ if (scriptName) {
+ if (!scriptsRecord.find(`${scriptName}.js`).isFile()) {
+ addTestError(test.testId, `Setup script does not exist: ${scriptName}.js`);
+ return '';
+ }
+
+ try {
+ const data = scriptsRecord.find(`${scriptName}.js`).text;
+ const lines = data.split(/\r?\n/);
+ lines.forEach(line => {
+ if (line.trim().length) script += '\t\t\t' + line.trim() + '\n';
+ });
+ } catch (err) {
+ log.warning(err);
+ }
+
+ scripts.push(`\t\t${scriptName}: function(testPageDocument){\n${script}\t\t}`);
+ }
+
+ return script;
+ }
+
+ function getSetupScriptDescription(desc) {
+ let str = '';
+ if (typeof desc === 'string') {
+ let d = desc.trim();
+ if (d.length) {
+ str = d;
+ }
+ }
+
+ return str;
+ }
+
+ function getScripts() {
+ let js = 'var scripts = {\n';
+ js += scripts.join(',\n');
+ js += '\n\t};';
+ return js;
+ }
+
+ let task = getTask(test.task);
+ let appliesTo = getAppliesToValues(test.appliesTo);
+ let mode = getModeValue(test.mode);
+
+ appliesTo.forEach(at => {
+ if (commands[task]) {
+ if (!commands[task][mode][at.toLowerCase()]) {
+ addTestError(
+ test.testId,
+ 'command is missing for the combination of task: "' +
+ task +
+ '", mode: "' +
+ mode +
+ '", and AT: "' +
+ at.toLowerCase() +
+ '" '
+ );
+ }
+ }
+ });
+
+ let assertions = [];
+ let id = test.testId;
+ if (parseInt(test.testId) < 10) {
+ id = '0' + id;
+ }
+
+ const cleanTaskName = cleanTask(test.task).replace(/\s+/g, '-');
+ let testFileName = `test-${id}-${cleanTaskName}-${mode}.html`;
+ let testJSONFileName = `test-${id}-${cleanTaskName}-${mode}.json`;
+
+ let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, testFileName);
+ let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, testJSONFileName);
+
+ let references = getReferences(refs.example, test.refs);
+ addSetupScript(test.setupScript);
+
+ for (let i = 1; i < 31; i++) {
+ if (!test['assertion' + i]) {
+ continue;
+ }
+ addAssertion(test['assertion' + i]);
+ }
+
+ /** @type {AriaATFile.Behavior} */
+ let testData = {
+ setup_script_description: getSetupScriptDescription(test.setupScriptDescription),
+ setupTestPage: test.setupScript,
+ applies_to: appliesTo,
+ mode: mode,
+ task: task,
+ assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion,
+ specific_user_instruction: test.instructions,
+ output_assertions: assertions,
+ };
+
+ emitFile(testPlanJsonFileBuildPath, JSON.stringify(testData, null, 2), 'utf8');
+
+ function getTestJson() {
+ return JSON.stringify(testData, null, 2);
+ }
+
+ function getCommandsJson() {
+ return beautify({ [task]: commands[task] }, null, 2, 40);
+ }
+
+ let testHTML = `
+
+
+
${test.title}
+${references}
+
+
+ `;
+
+ emitFile(testPlanHtmlFileBuildPath, testHTML, 'utf8');
+
+ /** @type {AriaATFile.CollectedTest} */
+ const collectedTest = {};
+
+ const applies_to_at = [];
+
+ allATKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0));
+
+ return [testFileName, applies_to_at];
+ }
+
+ /**
+ * Create an index file for a local server
+ * @param tasks
+ */
+ function createIndexFile(tasks, { emitFile }) {
+ let rows = '';
+ let all_ats = '';
+
+ allATNames.forEach(at => (all_ats += '' + at + ' \n'));
+
+ tasks.forEach(function (task) {
+ rows += `${task.id} `;
+ rows += `${task.title} `;
+ for (let i = 0; i < allATKeys.length; i++) {
+ if (task.applies_to_at[i]) {
+ rows += `${allATNames[i]} `;
+ } else {
+ rows += `not included `;
+ }
+ }
+ rows += `${task.script} \n`;
+ });
+
+ let indexHTML = `
+
+
+
+ Index of Assistive Technology Test Files
+
+
+
+
+ Index of Assistive Technology Test Files
+ This is useful for viewing the local files on a local web server.
+
+
+
+ Task ID
+ Testing Task
+ ${all_ats}
+ Setup Script Reference
+
+
+
+${rows}
+
+
+
+
+`;
+
+ emitFile(indexFileBuildOutputPath, indexHTML, 'utf8');
+ }
+
+ // Process CSV files
+ var refs = {};
+ var errorCount = 0;
+ var errors = '';
+ var indexOfURLs = [];
+ var checkedSourceHtmlScriptFiles = [];
+
+ function addTestError(id, error) {
+ errorCount += 1;
+ errors += '[Test ' + id + ']: ' + error + '\n';
+ }
+
+ function addCommandError({ testId, task }, key) {
+ errorCount += 1;
+ errors += `[Command]: The key reference "${key}" found in "${directory}/data/commands.csv" for "test id ${testId}: ${task}" is invalid. Command may not be defined in "tests/resources/keys.mjs".\n`;
+ }
+
+ const newTestPlan = newBuild.find(`tests/${path.basename(testPlanBuildDirectory)}`);
+ function emitFile(filepath, content) {
+ newTestPlan.add(path.relative(testPlanBuildDirectory, filepath), {
+ buffer: toBuffer(content),
+ });
+ }
+
+ function generateSourceHtmlScriptFile(filePath, content) {
+ // check that test plan's reference html file path is generated file
+ if (
+ filePath.includes('reference') &&
+ (filePath.split(path.sep).pop().match(/\./g) || []).length > 1
+ ) {
+ // generate file at `/tests//reference//.
+`,
+ button: reindent`
+
+
+ Run Test Setup
+
+`,
+ };
+}
diff --git a/lib/data/process-test-directory.js b/lib/data/process-test-directory.js
index 9e3d76110..d27079037 100644
--- a/lib/data/process-test-directory.js
+++ b/lib/data/process-test-directory.js
@@ -22,9 +22,11 @@ const { Queryable } = require('../util/queryable');
const { FileRecordChain } = require('../util/file-record-chain');
const { parseSupport } = require('./parse-support');
-const { parseTestCSVRow } = require('./parse-test-csv-row');
-const { parseCommandCSVRow } = require('./parse-command-csv-row');
-const { createCommandTuplesATModeTaskLookup } = require('./command-tuples-at-mode-task-lookup');
+const { parseTestCSVRowV2 } = require('./parse-test-csv-row');
+const { parseCommandCSVRowV2, flattenObject } = require('./parse-command-csv-row');
+const {
+ createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber,
+} = require('./command-tuples-at-mode-task-lookup');
const { renderHTML: renderCollectedTestHtml } = require('./templates/collected-test.html');
const { createExampleScriptsTemplate } = require('./example-scripts-template');
@@ -34,9 +36,6 @@ const { createExampleScriptsTemplate } = require('./example-scripts-template');
* @param {object} args={}
*/
const processTestDirectory = async ({ directory, args = {} }) => {
- let VERBOSE_CHECK = false;
- let VALIDATE_CHECK = false;
-
let suppressedMessages = 0;
/**
@@ -70,10 +69,8 @@ const processTestDirectory = async ({ directory, args = {} }) => {
};
// setup from arguments passed to npm script
- VERBOSE_CHECK = !!args.verbose;
- VALIDATE_CHECK = !!args.validate;
-
- const validModes = ['reading', 'interaction', 'item'];
+ const VERBOSE_CHECK = !!args.verbose;
+ const VALIDATE_CHECK = !!args.validate;
// cwd; @param rootDirectory is dependent on this file not moving from the `lib/data` folder
const libDataDirectory = path.dirname(__filename);
@@ -83,11 +80,12 @@ const processTestDirectory = async ({ directory, args = {} }) => {
const testPlanDirectory = path.join(rootDirectory, directory);
const resourcesDirectory = path.join(testsDirectory, 'resources');
-
- const supportFilePath = path.join(testsDirectory, 'support.json');
+ const supportJsonFilePath = path.join(testsDirectory, 'support.json');
+ const commandsJsonFilePath = path.join(testsDirectory, 'commands.json');
const testsCsvFilePath = path.join(testPlanDirectory, 'data', 'tests.csv');
- const atCommandsCsvFilePath = path.join(testPlanDirectory, 'data', 'commands.csv');
const referencesCsvFilePath = path.join(testPlanDirectory, 'data', 'references.csv');
+ const assertionsCsvFilePath = path.join(testPlanDirectory, 'data', 'assertions.csv');
+ const scriptsCsvFilePath = path.join(testPlanDirectory, 'data', 'scripts.csv');
// build output folders and file paths setup
const buildDirectory = path.join(rootDirectory, 'build');
@@ -106,14 +104,16 @@ const processTestDirectory = async ({ directory, args = {} }) => {
'tests/resources',
'tests/resources/*',
'tests/support.json',
+ 'tests/commands.json',
].join(','),
});
- const [testPlanRecord, resourcesOriginalRecord, supportRecord] = await Promise.all(
- [testPlanDirectory, resourcesDirectory, supportFilePath].map(filepath =>
- FileRecordChain.read(filepath)
- )
- );
+ const [testPlanRecord, resourcesOriginalRecord, supportJsonRecord, commandsJsonRecord] =
+ await Promise.all(
+ [testPlanDirectory, resourcesDirectory, supportJsonFilePath, commandsJsonFilePath].map(
+ filepath => FileRecordChain.read(filepath)
+ )
+ );
const scriptsRecord = testPlanRecord.find('data/js');
const resourcesRecord = resourcesOriginalRecord.filter({ glob: '{aria-at-*,keys,vrender}.mjs' });
@@ -128,19 +128,33 @@ const processTestDirectory = async ({ directory, args = {} }) => {
entries: testPlanRecord.filter({ glob: 'reference{,/**}' }).record.entries,
},
{ name: 'resources', ...resourcesRecord.record },
- { name: 'support.json', ...supportRecord.record },
+ { name: 'support.json', ...supportJsonRecord.record },
+ { name: 'commands.json', ...commandsJsonRecord.record },
],
},
],
});
- const keyDefs = {};
- const support = JSON.parse(supportRecord.text);
+ const supportJson = JSON.parse(supportJsonRecord.text);
+ const commandsJson = JSON.parse(commandsJsonRecord.text);
- let allATKeys = support.ats.map(({ key }) => key);
- let allATNames = support.ats.map(({ name }) => name);
+ const allAts = supportJson.ats;
+ const allAtKeys = allAts.map(({ key }) => key);
+ const allAtNames = allAts.map(({ name }) => name);
+ const atSettings = allAts
+ .map(({ settings }) => settings)
+ .flatMap(setting => Object.keys(setting));
- const validAppliesTo = ['Screen Readers', 'Desktop Screen Readers'].concat(allATKeys);
+ // Get paths to possible at keys
+ const atCommandsCsvFilePaths = getTestPlanDataFilePaths(
+ testPlanDirectory,
+ allAtKeys.map(key => `${key}-commands.csv`)
+ ).map((atCommandCsvFilePath, index) => ({ atKey: allAtKeys[index], atCommandCsvFilePath }));
+
+ // readingMode and interactionMode are known screen reader 'at modes' found in
+ // support.json at ats[].assertionTokens. The specific named modes are
+ // stored in ats[].settings
+ const validModes = ['readingMode', 'interactionMode', 'defaultMode'].concat(atSettings);
if (!testPlanRecord.isDirectory()) {
log.error(`The test directory '${testPlanDirectory}' does not exist. Check the path to tests.`);
@@ -150,158 +164,285 @@ const processTestDirectory = async ({ directory, args = {} }) => {
log.error(`The tests.csv file does not exist. Please create '${testsCsvFilePath}' file.`);
}
- if (!testPlanRecord.find('data/commands.csv').isFile()) {
+ if (!testPlanRecord.find('data/references.csv').isFile()) {
log.error(
- `The at-commands.csv file does not exist. Please create '${atCommandsCsvFilePath}' file.`
+ `The references.csv file does not exist. Please create '${referencesCsvFilePath}' file.`
);
}
- if (!testPlanRecord.find('data/references.csv').isFile()) {
+ if (!testPlanRecord.find('data/assertions.csv').isFile()) {
log.error(
- `The references.csv file does not exist. Please create '${referencesCsvFilePath}' file.`
+ `The assertions.csv file does not exist. Please create '${assertionsCsvFilePath}' file.`
);
}
- // get Keys that are defined
- try {
- // read contents of the file
- const keys = resourcesRecord.find('keys.mjs').text;
+ if (!testPlanRecord.find('data/scripts.csv').isFile()) {
+ log.error(`The scripts.csv file does not exist. Please create '${scriptsCsvFilePath}' file.`);
+ }
+
+ let someAtCommandsCsvExist = false;
+ let missingAtCommandsCsvFiles = [];
+ for (const atCommandCsvFilePath of atCommandsCsvFilePaths) {
+ const { atKey, atCommandCsvFilePath: filePath } = atCommandCsvFilePath;
- // split the contents by new line
- const lines = keys.split(/\r?\n/);
+ // Checks that at least one {at}-commands.csv exists
+ if (testPlanRecord.find(`data/${atKey}-commands.csv`).isFile()) someAtCommandsCsvExist = true;
+ else missingAtCommandsCsvFiles.push(filePath);
+ }
+ if (!someAtCommandsCsvExist) {
+ log.error(
+ `No *-commands.csv files found for ${directory}. Please create at least one of the following: ${missingAtCommandsCsvFiles.join(
+ ', '
+ )}`
+ );
+ }
- // print all lines
- lines.forEach(line => {
- let parts1 = line.split(' ');
- let parts2 = line.split('"');
+ /**
+ * @param {AriaATCSV.Reference} row
+ * @returns {AriaATCSV.Reference}
+ */
+ function validateReferencesKeys(row) {
+ if (
+ typeof row.refId !== 'string' ||
+ typeof row.type !== 'string' ||
+ typeof row.value !== 'string'
+ ) {
+ throw new Error('Row missing refId, type or value');
+ }
+ return row;
+ }
- if (parts1.length > 3) {
- let code = parts1[2].trim();
- keyDefs[code] = parts2[1].trim();
+ const validCommandKeys = /^(?:testId|command|settings|assertionExceptions|presentationNumber)$/;
+ const numericKeyFormat = /^_(\d+)$/;
+ const idFormat = /^[A-Za-z0-9-]+$/;
+ const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-]+\s*)+$/;
+ const settingsFormat = /^[A-Za-z0-9-\s]+$/;
+ function validateCommandsKeys(row) {
+ // example header:
+ // testId,command,settings,assertionExceptions,presentationNumber
+ for (const key of Object.keys(row)) {
+ if (numericKeyFormat.test(key)) {
+ throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
+ } else if (!validCommandKeys.test(key)) {
+ throw new Error(`Unknown *-commands.csv key: ${key} - check header row?`);
}
- });
- } catch (err) {
- log.warning(err);
+ }
+ if (!(row.testId?.length && row.command?.length && row.presentationNumber?.length)) {
+ throw new Error('Missing one of required testId, command, presentationNumber');
+ }
+ if (!idFormat.test(row.testId))
+ throw new Error('testId does not match the expected format: ' + row.testId);
+ if (row.settings && !settingsFormat.test(row.settings))
+ throw new Error('settings does not match the expected format: ' + row.settings);
+ if (row.assertionExceptions && !assertionsExceptionsFormat.test(row.assertionExceptions))
+ throw new Error(
+ 'assertionExceptions does not match the expectedFormat: ' + row.assertionExceptions
+ );
+ if (!Number(row.presentationNumber) > 0)
+ throw new Error(
+ 'presentationNumber does not match the expected format: ' + row.presentationNumber
+ );
+ return row;
}
- function cleanTask(task) {
- return task.replace(/'/g, '').replace(/;/g, '').trim().toLowerCase();
+ const validTestsKeys =
+ /^(?:testId|title|presentationNumber|setupScript|instructions|assertions)$/;
+ const titleFormat = /^[A-Z]([A-Za-z-',\s]){2,}[^.]$/;
+ function validateTestsKeys(row) {
+ // example header:
+ // testId,title,presentationNumber,setupScript,instructions,assertions
+ for (const key of Object.keys(row)) {
+ if (numericKeyFormat.test(key)) {
+ throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
+ } else if (!validTestsKeys.test(key)) {
+ throw new Error(`Unknown tests.csv key: ${key} - check header row?`);
+ }
+ }
+ if (
+ !(
+ row.testId?.length &&
+ row.title?.length &&
+ row.presentationNumber?.length &&
+ row.instructions?.length &&
+ row.assertions?.length
+ )
+ ) {
+ throw new Error(
+ 'Missing one of required testId, title, presentationNumber, instructions, assertions'
+ );
+ }
+ if (!idFormat.test(row.testId))
+ throw new Error('testId does not match the expected format: ' + row.testId);
+ if (!titleFormat.test(row.title))
+ throw new Error('title does not match the expected format: ' + row.title);
+ if (!Number(row.presentationNumber) > 0)
+ throw new Error(
+ 'presentationNumber does not match the expected format: ' + row.presentationNumber
+ );
+ return row;
+ }
+
+ const validAssertionsKeys =
+ /^(?:assertionId|priority|assertionStatement|assertionPhrase|refIds)$/;
+ const priorityFormat = /^[123]$/;
+ // TODO: The make-v2 script isn't removing the punctuations at the end which violates the requirements of this
+ // column's validation. Based on https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#assertionstatement
+ // Please ensure they are removing and use the commented regex instead.
+ const assertionStatementFormat = /^[A-Z].*[a-zA-Z'{}().](?![.!?])$/;
+ // const assertionStatementFormat = /^[A-Z].*[a-zA-Z'{}()](?![.!?])$/;
+
+ // TODO: Same as TODO above. Based on https://github.com/w3c/aria-at/wiki/Test-Format-Definition-V2#assertionphrase
+ const assertionPhraseFormat = /^[a-z].*[a-zA-Z'"{}()\d+.](?![.!?])$/;
+ // const assertionPhraseFormat = /^[a-z].*[a-zA-Z'"{}()\d+](?![.!?])$/;
+ function validateAssertionsKeys(row) {
+ // example header:
+ // assertionId,priority,assertionStatement,assertionPhrase,refIds
+ for (const key of Object.keys(row)) {
+ if (numericKeyFormat.test(key)) {
+ throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
+ } else if (!validAssertionsKeys.test(key)) {
+ throw new Error(`Unknown tests.csv key: ${key} - check header row?`);
+ }
+ }
+ if (
+ !(
+ row.assertionId?.length &&
+ row.priority?.length &&
+ row.assertionStatement?.length &&
+ row.assertionPhrase?.length
+ )
+ ) {
+ throw new Error(
+ 'Missing one of required assertionId, priority, assertionStatement, assertionPhrase'
+ );
+ }
+ if (!idFormat.test(row.assertionId))
+ throw new Error('assertionId does not match the expected format: ' + row.assertionId);
+ if (!priorityFormat.test(row.priority))
+ throw new Error('priority does not match the expected format: ' + row.priority);
+ if (!assertionStatementFormat.test(row.assertionStatement))
+ throw new Error(
+ 'assertionStatement does not match the expected format: ' + row.assertionStatement
+ );
+ if (!assertionPhraseFormat.test(row.assertionPhrase))
+ throw new Error('assertionPhrase does not match the expected format: ' + row.assertionPhrase);
+ return row;
+ }
+
+ const validScriptsKeys = /^(?:setupScript|setupScriptDescription)$/;
+ const setupScriptDescriptionFormat = /^[a-z].*[a-zA-Z'](?![.!?])$/;
+ function validateScriptsKeys(row) {
+ // example header:
+ // setupScript,setupScriptDescription
+ for (const key of Object.keys(row)) {
+ if (numericKeyFormat.test(key)) {
+ throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
+ } else if (!validScriptsKeys.test(key)) {
+ throw new Error(`Unknown tests.csv key: ${key} - check header row?`);
+ }
+ }
+ if (!(row.setupScript?.length && row.setupScriptDescription?.length)) {
+ throw new Error('Missing one of required setupScript, setupScriptDescription');
+ }
+ if (!setupScriptDescriptionFormat.test(row.setupScriptDescription))
+ throw new Error(
+ 'setupScriptDescription does not match the expected format: ' + row.setupScriptDescription
+ );
+ return row;
+ }
+
+ const [referencesCsv, testsCsv, assertionsCsv, scriptsCsv, ...atCommandsCsvs] = await Promise.all(
+ [
+ readCSVFile('data/references.csv', validateReferencesKeys),
+ readCSVFile('data/tests.csv', validateTestsKeys),
+ readCSVFile('data/assertions.csv', validateAssertionsKeys),
+ readCSVFile('data/scripts.csv', validateScriptsKeys),
+ ...atCommandsCsvFilePaths.map(({ atKey }) =>
+ readCSVFile(`data/${atKey}-commands.csv`, validateCommandsKeys)
+ ),
+ ]
+ );
+
+ // Works because the order of allAtKeys and atCommandsCsvs should be maintained up this point
+ const parsedAtCommandsCsvs = allAtKeys.map((atKey, index) => ({
+ atKey,
+ commands: atCommandsCsvs[index],
+ }));
+
+ const testsParsed = parseTestCSVRowV2({
+ tests: testsCsv,
+ assertions: assertionsCsv,
+ scripts: scriptsCsv,
+ commands: parsedAtCommandsCsvs,
+ });
+
+ /**
+ *
+ * @param {string} root
+ * @param {string[]} fileNames
+ * @return {array[string]}
+ */
+ function getTestPlanDataFilePaths(root, fileNames = []) {
+ let filePaths = [];
+
+ for (const fileName of fileNames) {
+ const filePath = path.join(testPlanDirectory, 'data', fileName);
+ filePaths.push(filePath);
+ }
+
+ return filePaths;
}
/**
* Create Test File
* @param {AriaATCSV.Test} test
- * @param refs
- * @param commands
+ * @param {Object>} refs
+ * @param {Object>>} atCommandsMap
+ * @param {object} options
+ * @param {function(string, string)} options.addTestError
+ * @param {function(filePath: string, content: any, encoding: string)} options.emitFile
+ * @param {FileRecordChain} options.scriptsRecord
+ * @param {Queryable} options.exampleScriptedFilesQueryable
* @returns {(string|*[])[]}
*/
function createTestFile(
test,
refs,
- commands,
+ atCommandsMap,
{ addTestError, emitFile, scriptsRecord, exampleScriptedFilesQueryable }
) {
let scripts = [];
// default setupScript if test has undefined setupScript
- if (!scriptsRecord.find(`${test.setupScript}.js`).isFile()) test.setupScript = '';
-
- function getModeValue(value) {
- let v = value.trim().toLowerCase();
- if (!validModes.includes(v)) {
- addTestError(test.testId, '"' + value + '" is not valid value for "mode" property.');
- }
- return v;
- }
-
- function getTask(t) {
- let task = cleanTask(t);
-
- if (typeof commands[task] !== 'object') {
- addTestError(test.testId, '"' + task + '" does not exist in commands.csv file.');
- }
-
- return task;
- }
+ if (!scriptsRecord.find(`${test.setupScript?.script}.js`).isFile())
+ test.setupScript = {
+ ...test.setupScript,
+ script: '',
+ };
- function getAppliesToValues(values) {
- function checkValue(value) {
- let v1 = value.trim().toLowerCase();
- for (let i = 0; i < validAppliesTo.length; i++) {
- let v2 = validAppliesTo[i];
- if (v1 === v2.toLowerCase()) {
- return v2;
- }
- }
- return false;
+ function getTestId(testId) {
+ if (typeof atCommandsMap[testId] !== 'object') {
+ addTestError(test.testId, '"' + testId + '" does not exist in *-commands.csv file.');
}
-
- // check for individual assistive technologies
- let items = values.split(',');
- let newValues = [];
- items.filter(item => {
- let value = checkValue(item);
- if (!value) {
- addTestError(test.testId, '"' + item + '" is not valid value for "appliesTo" property.');
- }
-
- newValues.push(value);
- });
-
- return newValues;
+ return testId;
}
- /**
- * Determines priority level (default is 1) of assertion string, then adds it to the collection of assertions for
- * the test plan
- * @param {string} a - Assertion string to be evaluated
- */
- function addAssertion(a) {
- let level = '1';
- let str = a;
- a = a.trim();
-
- // matches a 'colon' when preceded by either of the digits 1 OR 2 (SINGLE CHARACTER), at the start of the string
- let parts = a.split(/(?<=^[1-2]):/g);
-
- if (parts.length === 2) {
- level = parts[0];
- str = parts[1].substring(0);
- if (level !== '1' && level !== '2') {
- addTestError(
- test.testId,
- "Level value must be 1 or 2, value found was '" +
- level +
- "' for assertion '" +
- str +
- "' (NOTE: level 2 defined for this assertion)."
- );
- level = '2';
- }
- }
-
- if (a.length) {
- assertions.push([level, str]);
+ function getModeValue({ settings }) {
+ let v = settings.trim();
+ if (!validModes.includes(v)) {
+ addTestError(test.testId, '"' + settings + '" is not valid value for "settings" property.');
}
+ return v;
}
- function getReferences(example, testRefs) {
+ function getReferences(test, refs) {
let links = '';
- if (typeof example === 'string' && example.length) {
- links += ` \n`;
- }
+ test.references.forEach(({ refId }) => {
+ const { value: link, linkText } = refs[refId];
- let items = test.refs.split(' ');
- items.forEach(function (item) {
- item = item.trim();
-
- if (item.length) {
- if (typeof refs[item] === 'string') {
- links += ` \n`;
- } else {
- addTestError(test.testId, 'Reference does not exist: ' + item);
- }
+ if (typeof link === 'string' && link.length) {
+ links += ` \n`;
}
});
@@ -345,65 +486,51 @@ const processTestDirectory = async ({ directory, args = {} }) => {
}
function getScripts() {
- let js = 'var scripts = {\n';
+ let js = 'let scripts = {\n';
js += scripts.join(',\n');
js += '\n\t};';
return js;
}
- let task = getTask(test.task);
- let appliesTo = getAppliesToValues(test.appliesTo);
- let mode = getModeValue(test.mode);
+ let testId = getTestId(test.testId);
+ let modes = test.target.ats.map(getModeValue).join('_');
- appliesTo.forEach(at => {
- if (commands[task]) {
- if (!commands[task][mode][at.toLowerCase()]) {
+ test.target.ats.forEach(at => {
+ if (atCommandsMap[testId]) {
+ if (!atCommandsMap[testId][at.settings][at.key]) {
addTestError(
- test.testId,
+ testId,
'command is missing for the combination of task: "' +
- task +
+ testId +
'", mode: "' +
- mode +
+ atCommandsMap[testId][at.settings] +
'", and AT: "' +
- at.toLowerCase() +
+ at.key.toLowerCase() +
'" '
);
}
}
});
- let assertions = [];
- let id = test.testId;
- if (parseInt(test.testId) < 10) {
- id = '0' + id;
- }
-
- const cleanTaskName = cleanTask(test.task).replace(/\s+/g, '-');
- let testFileName = `test-${id}-${cleanTaskName}-${mode}.html`;
- let testJSONFileName = `test-${id}-${cleanTaskName}-${mode}.json`;
-
- let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, testFileName);
- let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, testJSONFileName);
+ // testId and test level presentationNumber should be enough of a descriptor to differentiate the values
+ let testFileName = `test-${test.presentationNumber.toString().padStart(2, '0')}-${testId}`;
+ let testPlanHtmlFileBuildPath = path.join(testPlanBuildDirectory, `${testFileName}.html`);
+ let testPlanJsonFileBuildPath = path.join(testPlanBuildDirectory, `${testFileName}.json`);
- let references = getReferences(refs.example, test.refs);
- addSetupScript(test.setupScript);
-
- for (let i = 1; i < 31; i++) {
- if (!test['assertion' + i]) {
- continue;
- }
- addAssertion(test['assertion' + i]);
- }
+ let exampleReferences = getReferences(test, refs);
+ addSetupScript(test.setupScript.script);
/** @type {AriaATFile.Behavior} */
let testData = {
- setup_script_description: getSetupScriptDescription(test.setupScriptDescription),
- setupTestPage: test.setupScript,
- applies_to: appliesTo,
- mode: mode,
- task: task,
+ task: testId,
+ mode: modes,
+ applies_to: [...new Set(test.target.ats.map(({ key }) => key))],
+ setup_script_description: getSetupScriptDescription(test.setupScript.scriptDescription),
+ setupTestPage: test.setupScript.script,
specific_user_instruction: test.instructions,
- output_assertions: assertions,
+ assertionResponseQuestion: supportJson.testPlanStrings.assertionResponseQuestion,
+ commandsInfo: test.commandsInfo,
+ output_assertions: test.assertions,
};
emitFile(testPlanJsonFileBuildPath, JSON.stringify(testData, null, 2), 'utf8');
@@ -413,14 +540,14 @@ const processTestDirectory = async ({ directory, args = {} }) => {
}
function getCommandsJson() {
- return beautify({ [task]: commands[task] }, null, 2, 40);
+ return beautify({ [testId]: atCommandsMap[testId] }, null, 2, 40);
}
let testHTML = `
${test.title}
-${references}
+${exampleReferences}
@@ -430,14 +557,20 @@ ${references}
new Promise((resolve) => {
fetch('../support.json')
.then(response => resolve(response.json()))
+ }).then(supportJson => {
+ return fetch('../commands.json')
+ .then(response => response.json())
+ .then(allCommandsJson => ({ supportJson, allCommandsJson })
+ );
})
- .then(supportJson => {
+ .then(({ supportJson, allCommandsJson }) => {
const testJson = ${getTestJson()};
const commandJson = ${getCommandsJson()};
- initialize(supportJson, commandJson);
+ initialize(supportJson, commandJson, allCommandsJson);
verifyATBehavior(testJson);
displayTestPageAndInstructions(${JSON.stringify(
- exampleScriptedFilesQueryable.where({ name: test.setupScript ? test.setupScript : '' }).path
+ exampleScriptedFilesQueryable.where({ name: test.setupScript ? test.setupScript.script : '' })
+ .path
)});
});
@@ -445,37 +578,44 @@ ${references}
emitFile(testPlanHtmlFileBuildPath, testHTML, 'utf8');
- /** @type {AriaATFile.CollectedTest} */
- const collectedTest = {};
-
const applies_to_at = [];
-
- allATKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0));
+ allAtKeys.forEach(at => applies_to_at.push(testData.applies_to.indexOf(at) >= 0));
return [testFileName, applies_to_at];
}
/**
* Create an index file for a local server
- * @param tasks
+ * @param {object[]} tasks
+ * @param {number} tasks.seq
+ * @param {string} tasks.id
+ * @param {string} tasks.title
+ * @param {string} tasks.href
+ * @param {object} tasks.script
+ * @param {string} tasks.script.script
+ * @param {string} tasks.script.scriptDescription
+ * @param {boolean[]} tasks.applies_to_at
+ * @param {object} options
+ * @param {function(filePath: string, content: any, encoding: string)} options.emitFile
*/
function createIndexFile(tasks, { emitFile }) {
let rows = '';
let all_ats = '';
- allATNames.forEach(at => (all_ats += '' + at + ' \n'));
+ allAtNames.forEach(at => (all_ats += '' + at + ' \n'));
tasks.forEach(function (task) {
- rows += `${task.id} `;
+ rows += `${task.seq} `;
+ rows += `${task.id} `;
rows += `${task.title} `;
- for (let i = 0; i < allATKeys.length; i++) {
+ for (let i = 0; i < allAtKeys.length; i++) {
if (task.applies_to_at[i]) {
- rows += `${allATNames[i]} `;
+ rows += `${allAtNames[i]} `;
} else {
rows += `not included `;
}
}
- rows += `${task.script} \n`;
+ rows += `${task.script.script} \n`;
});
let indexHTML = `
@@ -488,7 +628,7 @@ ${references}
display: table;
border-collapse: collapse;
border-spacing: 2px;
- border-color: gray;
+ border-color: rgb(128,128,128);
}
thead {
@@ -500,7 +640,7 @@ ${references}
tbody {
display: table-row-group;
vertical-align: middle;
- border-color: gray;
+ border-color: rgb(128,128,128);
}
tr:nth-child(even) {background: #DDD}
@@ -509,7 +649,7 @@ ${references}
tr {
display: table-row;
vertical-align: inherit;
- border-color: gray;
+ border-color: rgb(128,128,128);
}
td {
@@ -539,6 +679,7 @@ ${references}
+ Sequence
Task ID
Testing Task
${all_ats}
@@ -557,20 +698,57 @@ ${rows}
}
// Process CSV files
- var refs = {};
- var errorCount = 0;
- var errors = '';
- var indexOfURLs = [];
- var checkedSourceHtmlScriptFiles = [];
+ let refs = {};
+ let errorCount = 0;
+ let errors = '';
+ let indexOfURLs = [];
+ let checkedSourceHtmlScriptFiles = [];
+ /**
+ * @param {string} id
+ * @param {string} error
+ */
function addTestError(id, error) {
errorCount += 1;
errors += '[Test ' + id + ']: ' + error + '\n';
}
- function addCommandError({ testId, task }, key) {
+ /**
+ * @param {object} options
+ * @param {string} options.testId
+ * @param {object} options.target
+ * @param {string} key
+ */
+ function addCommandError(
+ {
+ testId,
+ target: {
+ at: { key: atKey },
+ },
+ },
+ key
+ ) {
+ errorCount += 1;
+ errors += `[Command]: The key reference "${key}" found in "${directory}/data/${atKey}-commands.csv" for "test id ${testId}" is invalid. Command may not be defined in "tests/commands.json".\n`;
+ }
+
+ /**
+ * @param {object} options
+ * @param {string} options.testId
+ * @param {object} options.target
+ * @param {string} assertion
+ */
+ function addCommandAssertionExceptionError(
+ {
+ testId,
+ target: {
+ at: { key: atKey },
+ },
+ },
+ assertion
+ ) {
errorCount += 1;
- errors += `[Command]: The key reference "${key}" found in "${directory}/data/commands.csv" for "test id ${testId}: ${task}" is invalid. Command may not be defined in "tests/resources/keys.mjs".\n`;
+ errors += `[Command]: assertionExceptions reference "${assertion}" found in "${directory}/data/${atKey}-commands.csv" for "test id ${testId}" is invalid.\n`;
}
const newTestPlan = newBuild.find(`tests/${path.basename(testPlanBuildDirectory)}`);
@@ -595,7 +773,14 @@ ${rows}
// intended to be an internal helper to reduce some code duplication and make logging for csv errors simpler
async function readCSVFile(filePath, rowValidator = identity => identity) {
- const rawCSV = await readCSV(testPlanRecord.find(filePath));
+ let rawCSV = [];
+ try {
+ rawCSV = await readCSV(testPlanRecord.find(filePath));
+ } catch (e) {
+ log.warning(`WARNING: Error reading ${path.join(testPlanDirectory, filePath)}`);
+ return rawCSV;
+ }
+
let index = 0;
function printError(message) {
// line number is index+2
@@ -627,101 +812,73 @@ ${rows}
return rawCSV;
}
- function validateReferencesKeys(row) {
- if (typeof row.refId !== 'string' || typeof row.value !== 'string') {
- throw new Error('Row missing refId or value');
- }
- return row;
- }
+ for (const row of referencesCsv) {
+ const {
+ references: { aria, htmlAam },
+ } = supportJson;
- const validCommandKeys = /^(?:testId|task|mode|at|command[A-Z])$/;
- const numericKeyFormat = /^_(\d+)$/;
- function validateCommandsKeys(row) {
- // example header:
- // testId,task,mode,at,commandA,commandB,commandC,commandD,commandE,commandF
- for (const key of Object.keys(row)) {
- if (numericKeyFormat.test(key)) {
- throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
- } else if (!validCommandKeys.test(key)) {
- throw new Error(`Unknown commands.csv key: ${key} - check header row?`);
- }
- }
- if (
- !(
- row.testId?.length &&
- row.task?.length &&
- row.mode?.length &&
- row.at?.length &&
- row.commandA?.length
- )
- ) {
- throw new Error('Missing one of required testId, task, mode, at, commandA');
- }
- return row;
- }
+ let refId = row.refId.trim();
+ let type = row.type.trim();
+ let value = row.value.trim();
+ let linkText = row.linkText.trim();
- const validTestsKeys =
- /^(?:testId|title|appliesTo|mode|task|setupScript|setupScriptDescription|refs|instructions|assertion(?:[1-9]|[1-2][0-9]|30))$/;
- function validateTestsKeys(row) {
- // example header:
- // testId,title,appliesTo,mode,task,setupScript,setupScriptDescription,refs,instructions,assertion1,assertion2,assertion3,assertion4,assertion5,assertion6,assertion7
- for (const key of Object.keys(row)) {
- if (numericKeyFormat.test(key)) {
- throw new Error(`Column found without header row, ${+key.substring(1) + 1}`);
- } else if (!validTestsKeys.test(key)) {
- throw new Error(`Unknown tests.csv key: ${key} - check header row?`);
- }
- }
- if (
- !(
- row.testId?.length &&
- row.title?.length &&
- row.appliesTo?.length &&
- row.mode?.length &&
- row.task?.length
- )
- ) {
- throw new Error('Missing one of required testId, title, appliesTo, mode, task');
+ if (type === 'aria') {
+ value = `${aria.baseUrl}${aria.fragmentIds[value]}`;
+ linkText = `${linkText} ${aria.linkText}`;
}
- return row;
- }
- const [refRows, atCommands, tests] = await Promise.all([
- readCSVFile('data/references.csv', validateReferencesKeys),
- readCSVFile('data/commands.csv', validateCommandsKeys),
- readCSVFile('data/tests.csv', validateTestsKeys),
- ]);
+ if (type === 'htmlAam') {
+ value = `${htmlAam.baseUrl}${htmlAam.fragmentIds[value]}`;
+ linkText = `${linkText} ${htmlAam.linkText}`;
+ }
- for (const row of refRows) {
- refs[row.refId] = row.value.trim();
+ refs[refId] = {
+ type,
+ value,
+ linkText,
+ };
}
- const scripts = loadScripts(scriptsRecord);
+ const scriptsSource = loadScriptsSource(scriptsRecord);
+ const commandsParsed = parseCommandCSVRowV2(
+ {
+ commands: parsedAtCommandsCsvs,
+ },
+ commandsJson
+ );
+ const {
+ references: { aria, htmlAam },
+ } = supportJson;
+ const referencesParsed = parseReferencesCSV(referencesCsv, { aria, htmlAam });
- const commandsParsed = atCommands.map(parseCommandCSVRow);
- const testsParsed = tests.map(parseTestCSVRow);
- const referencesParsed = parseReferencesCSV(refRows);
+ const keyDefs = flattenObject(commandsJson);
const keysParsed = parseKeyMap(keyDefs);
- const supportParsed = parseSupport(support);
+ const supportParsed = parseSupport(supportJson);
+ // TODO: This causes 'delete' to incorrectly not be recognized as a key, why?
const keysValidated = validateKeyMap(keysParsed, {
addKeyMapError(reason) {
errorCount += 1;
- errors += `[resources/keys.mjs]: ${reason}\n`;
+ errors += `[commands.json]: ${reason}\n`;
},
});
const supportQueryables = {
- at: Queryable.from('at', supportParsed.ats),
- atGroup: Queryable.from('atGroup', supportParsed.atGroups),
+ ats: Queryable.from('ats', supportParsed.ats),
+ atGroups: Queryable.from('atGroups', supportParsed.atGroups),
+ references: Queryable.from('references', supportParsed.ats),
+ testPlanStrings: Queryable.from('testPlanStrings', supportParsed.ats),
};
+
const keyQueryable = Queryable.from('key', keysValidated);
+ const assertionQueryables = Queryable.from('assertions', assertionsCsv);
const commandLookups = {
key: keyQueryable,
support: supportQueryables,
+ assertions: assertionQueryables,
};
const commandsValidated = commandsParsed.map(command =>
- validateCommand(command, commandLookups, { addCommandError })
+ validateCommand(command, commandLookups, { addCommandError, addCommandAssertionExceptionError })
);
const referenceQueryable = Queryable.from('reference', referencesParsed);
@@ -740,8 +897,8 @@ ${rows}
const testLookups = {
command: Queryable.from('command', commandsValidated),
mode: Queryable.from('mode', validModes),
+ script: Queryable.from('script', scriptsSource),
reference: referenceQueryable,
- script: Queryable.from('script', scripts),
support: supportQueryables,
};
const testsValidated = testsParsed.map(test =>
@@ -760,42 +917,120 @@ ${rows}
reason => log.warning(`[${examplePathOriginal}]: ${reason.message}`),
() => createExampleScriptsTemplate(exampleRecord)
);
- const exampleScriptedFiles = [{ name: '', source: '' }, ...scripts].map(({ name, source }) => ({
- name,
- path: examplePathTemplate(name),
- content: exampleTemplate.render(exampleTemplateParams(name, source)).toString(),
- }));
+ const exampleScriptedFiles = [{ name: '', source: '' }, ...scriptsSource].map(
+ ({ name, source }) => ({
+ name,
+ path: examplePathTemplate(name),
+ content: exampleTemplate.render(exampleTemplateParams(name, source)).toString(),
+ })
+ );
const exampleScriptedFilesQueryable = Queryable.from('example', exampleScriptedFiles);
-
const commandQueryable = Queryable.from('command', commandsValidated);
const testsCollected = testsValidated.flatMap(test => {
- return test.target.at.map(({ key }) =>
+ return test.target.ats.map(({ key, settings }) =>
collectTestData({
test,
command: commandQueryable.where({
testId: test.testId,
- target: { at: { key } },
+ target: { at: { key, settings } },
}),
reference: referenceQueryable,
+ supportAt: supportQueryables.ats,
example: exampleScriptedFilesQueryable,
- key: keyQueryable,
- modeInstructionTemplate: MODE_INSTRUCTION_TEMPLATES_QUERYABLE,
+ modeInstructionTemplate: MODE_INSTRUCTION_TEMPLATES_QUERYABLE(supportJson),
})
);
});
+ // So there can be a single {at}.collected.(json|html) instead of
+ // multiple {at}-{settings}.collected.(json|html)
+ let combinedCollectedTests = [];
+ const _testsCollected = JSON.parse(JSON.stringify(testsCollected.slice(0)));
+
+ // { eg. jaws: [ ... ], nvda: [ ... ], vo: [ ... ] }
+ const collectedTestsGroupedByAt = {};
+ allAtKeys.forEach(atKey => {
+ collectedTestsGroupedByAt[atKey] = _testsCollected.filter(
+ collected => collected.target.at.key === atKey
+ );
+ });
+
+ for (const key in collectedTestsGroupedByAt) {
+ const collectedTestAtGroup = collectedTestsGroupedByAt[key];
+
+ // Get the unique testIds found for the at
+ const collectedTestsGroupedByAtTestIds = [
+ ...new Set(collectedTestAtGroup.map(c => c.info.testId)),
+ ];
+
+ // Group by the testIds
+ for (const testId of collectedTestsGroupedByAtTestIds) {
+ const collectedTestsGroupedByAtGroupedByTestIds = collectedTestAtGroup.filter(
+ c => c.info.testId === testId
+ );
+
+ // Declare a common collectedTest that will be modified (if needed)
+ let common = collectedTestsGroupedByAtGroupedByTestIds[0];
+ const settings = common.target.at.settings;
+ common = {
+ ...common,
+ instructions: {
+ ...common.instructions,
+ mode: { [settings]: common.instructions.mode },
+ },
+ commands: common.commands.map(command => ({
+ ...command,
+ settings,
+ })),
+ };
+
+ // Iterate over the remaining collected tests
+ if (collectedTestsGroupedByAtGroupedByTestIds.length > 1) {
+ for (const additional of collectedTestsGroupedByAtGroupedByTestIds.slice(1)) {
+ const settings = additional.target.at.settings;
+
+ common.target.at.settings = `${common.target.at.settings}_${settings}`;
+ common.instructions.mode[settings] = additional.instructions.mode;
+ common.commands.push(
+ ...additional.commands.map(command => ({
+ ...command,
+ settings,
+ }))
+ );
+ }
+ }
+
+ combinedCollectedTests.push(common);
+ }
+ }
+
const files = [
- ...createScriptFiles(scripts, testPlanBuildDirectory),
+ ...createScriptFiles(scriptsSource, testPlanBuildDirectory),
...exampleScriptedFiles.map(({ path: pathSuffix, content }) => ({
path: path.join('build', 'tests', path.basename(directory), pathSuffix),
content,
})),
- ...testsCollected.map(collectedTest =>
- createCollectedTestFile(collectedTest, testPlanBuildDirectory)
- ),
- ...testsCollected.map(collectedTest =>
- createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory)
- ),
+ // TODO: If there is a need to individually view jaws-pcCursor or nvda-browseMode for example
+ // ...testsCollected.map(collectedTest =>
+ // createCollectedTestFile(collectedTest, testPlanBuildDirectory)
+ // ),
+ // ...testsCollected.map(collectedTest =>
+ // createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory)
+ // ),
+ ...combinedCollectedTests.map(collectedTest => {
+ const presentationNumber = parseInt(collectedTest.info.presentationNumber)
+ .toString()
+ .padStart(2, '0');
+ const fileName = `test-${presentationNumber}-${collectedTest.info.testId}-${collectedTest.target.at.key}`;
+ return createCollectedTestFile(collectedTest, testPlanBuildDirectory, fileName);
+ }),
+ ...combinedCollectedTests.map(collectedTest => {
+ const presentationNumber = parseInt(collectedTest.info.presentationNumber)
+ .toString()
+ .padStart(2, '0');
+ const fileName = `test-${presentationNumber}-${collectedTest.info.testId}-${collectedTest.target.at.key}`;
+ return createCollectedTestHtmlFile(collectedTest, testPlanBuildDirectory, fileName);
+ }),
];
files.forEach(file => {
generateSourceHtmlScriptFile(file.path, file.content);
@@ -819,16 +1054,18 @@ ${rows}
});
}
- const atCommandsMap = createCommandTuplesATModeTaskLookup(commandsValidated);
+ const atCommandsMap =
+ createAtCommandTuplesATSettingsTestIdLookupByPresentationNumber(commandsValidated);
+
emitFile(
path.join(testPlanBuildDirectory, 'commands.json'),
beautify(atCommandsMap, null, 2, 40)
);
log('Creating the following test files: ');
- tests.forEach(function (test) {
+ testsParsed.forEach(function (testParsed, index) {
try {
- const [url, applies_to_at] = createTestFile(test, refs, atCommandsMap, {
+ const [url, applies_to_at] = createTestFile(testParsed, refs, atCommandsMap, {
addTestError,
emitFile,
scriptsRecord,
@@ -836,14 +1073,15 @@ ${rows}
});
indexOfURLs.push({
- id: test.testId,
- title: test.title,
+ seq: index + 1,
+ id: testParsed.testId,
+ title: testParsed.title,
href: url,
- script: test.setupScript,
+ script: testParsed.setupScript,
applies_to_at: applies_to_at,
});
- log('[Test ' + test.testId + ']: ' + url);
+ log('[Test ' + testParsed.testId + ']: ' + url);
} catch (err) {
log.warning(err);
}
@@ -870,7 +1108,7 @@ ${rows}
);
log.warning(errors);
} else {
- log('No validation errors detected\n');
+ log(`No validation errors detected for ${directory}\n`);
}
return { isSuccessfulRun: errorCount === 0, suppressedMessages };
@@ -917,7 +1155,7 @@ function readCSV(record) {
* @param {FileRecordChain} testPlanJS
* @returns {AriaATParsed.ScriptSource[]}
*/
-function loadScripts(testPlanJS) {
+function loadScriptsSource(testPlanJS) {
return testPlanJS.filter({ glob: ',*.js' }).entries.map(({ name: fileName, text: source }) => {
const name = path.basename(fileName, '.js');
const modulePath = path.posix.join('scripts', `${name}.module.js`);
@@ -990,25 +1228,45 @@ function createScriptFiles(scripts, testPlanBuildDirectory) {
}
/**
- * @param {Object} keyLines
+ * @param {Object} keyDefs
* @returns {AriaATParsed.KeyMap}
*/
function parseKeyMap(keyDefs) {
const keyMap = {};
for (const id in keyDefs) {
- keyMap[id] = { id, keystroke: keyDefs[id] };
+ if (id.includes('.')) {
+ const [type, key] = id.split('.');
+ keyMap[key] = { id: key, type, keystroke: keyDefs[id] };
+ }
}
return keyMap;
}
/**
* @param {AriaATCSV.Reference[]} referenceRows
+ * @param {AriaATCSV.SupportReference} aria
+ * @param {AriaATCSV.SupportReference} htmlAam
* @returns {AriaATParsed.ReferenceMap}
*/
-function parseReferencesCSV(referenceRows) {
+function parseReferencesCSV(referenceRows, { aria, htmlAam }) {
const refMap = {};
- for (const { refId, value } of referenceRows) {
- refMap[refId] = { refId, value: value.trim() };
+ for (const { refId: _refId, type: _type, value: _value, linkText: _linkText } of referenceRows) {
+ let refId = _refId?.trim();
+ let type = _type?.trim();
+ let value = _value?.trim();
+ let linkText = _linkText?.trim();
+
+ if (type === 'aria') {
+ value = `${aria.baseUrl}${aria.fragmentIds[value]}`;
+ linkText = `${linkText} ${aria.linkText}`;
+ }
+
+ if (type === 'htmlAam') {
+ value = `${htmlAam.baseUrl}${htmlAam.fragmentIds[value]}`;
+ linkText = `${linkText} ${htmlAam.linkText}`;
+ }
+
+ refMap[refId] = { refId, type, value, linkText };
}
return refMap;
}
@@ -1019,34 +1277,62 @@ function parseReferencesCSV(referenceRows) {
* @param {Queryable} data.key
* @param {object} data.support
* @param {Queryable<{key: string, name: string}>} data.support.at
- * @param {object} [options]
- * @param {function(AriaATParsed.Command, string): void} [options.addCommandError]
+ * @param {object} options
+ * @param {function(AriaATParsed.Command, string): void} options.addCommandError
+ * @param {function(AriaATParsed.Command, string): void} options.addCommandAssertionExceptionError
* @returns {AriaATValidated.Command}
*/
-function validateCommand(commandParsed, data, { addCommandError = () => {} } = {}) {
+function validateCommand(
+ commandParsed,
+ data,
+ { addCommandError = () => {}, addCommandAssertionExceptionError = () => {} } = {}
+) {
return {
...commandParsed,
target: {
...commandParsed.target,
at: {
...commandParsed.target.at,
- ...mapDefined(data.support.at.where({ key: commandParsed.target.at.key }), ({ name }) => ({
+ ...mapDefined(data.support.ats.where({ key: commandParsed.target.at.key }), ({ name }) => ({
name,
})),
},
},
- commands: commandParsed.commands.map(({ id, keypresses: commandKeypresses, ...rest }) => {
- const keypresses = commandKeypresses.map(keypress => {
- const key = data.key.where(keypress);
- if (!key) {
- addCommandError(commandParsed, keypress.id);
+ commands: commandParsed.commands.map(({ id, keypresses, assertionExceptions, ...rest }) => {
+ keypresses.forEach(keypress => {
+ if (keypress.id.includes('+')) {
+ const splitKeys = keypress.id.split('+');
+ splitKeys.forEach(splitKey => {
+ const key = data.key.where({ id: splitKey });
+ if (!key) {
+ addCommandError(commandParsed, keypress.id);
+ }
+ });
+ } else {
+ const key = data.key.where({ id: keypress.id });
+ if (!key) {
+ addCommandError(commandParsed, keypress.id);
+ }
}
- return key || {};
});
+
return {
- id: id,
- keystroke: keypresses.map(({ keystroke }) => keystroke).join(', then '),
+ id,
keypresses,
+ assertionExceptions: assertionExceptions.map(each => {
+ const [_priority, assertionId] = each.split(':');
+ const priority = Number(_priority);
+
+ if (isNaN(priority)) {
+ addCommandAssertionExceptionError(commandParsed, each);
+ } else {
+ if (!/^[0123]$/.test(_priority)) {
+ addCommandAssertionExceptionError(commandParsed, each);
+ }
+ }
+
+ return { priority, assertionId };
+ }),
...rest,
};
}),
@@ -1055,83 +1341,44 @@ function validateCommand(commandParsed, data, { addCommandError = () => {} } = {
/**
* @param {AriaATParsed.KeyMap} keyMap
+ * @param {object} options
+ * @param {function(string): void} options.addKeyMapError
*/
function validateKeyMap(keyMap, { addKeyMapError }) {
- if (!keyMap.ALT_DELETE) {
- addKeyMapError(`ALT_DELETE is not defined in keys module.`);
- }
- if (!keyMap.INS_Z) {
- addKeyMapError(`INS_Z is not defined in keys module.`);
- }
- if (!keyMap.ESC) {
- addKeyMapError(`ESC is not defined in keys module.`);
- }
- if (!keyMap.INS_SPACE) {
- addKeyMapError(`INS_SPACE is not defined in keys module.`);
- }
- if (!keyMap.LEFT) {
- addKeyMapError(`LEFT is not defined in keys module.`);
- }
- if (!keyMap.RIGHT) {
- addKeyMapError(`RIGHT is not defined in keys module.`);
- }
+ if (!keyMap.alt) addKeyMapError('"alt" is not defined in keys module.');
+ if (!keyMap.del) addKeyMapError('"del" is not defined in keys module.');
+ if (!keyMap.ins) addKeyMapError('"ins" is not defined in keys module.');
+ if (!keyMap.z) addKeyMapError('"z" is not defined in keys module.');
+ if (!keyMap.esc) addKeyMapError('"esc" is not defined in keys module.');
+ if (!keyMap.space) addKeyMapError('"space" is not defined in keys module.');
+ if (!keyMap.left) addKeyMapError('"left" is not defined in keys module.');
+ if (!keyMap.right) addKeyMapError('"right" is not defined in keys module.');
+
return keyMap;
}
-const MODE_INSTRUCTION_TEMPLATES_QUERYABLE = Queryable.from('modeInstructionTemplate', [
- {
- at: 'jaws',
- mode: 'reading',
- render: data => {
- const altDelete = data.key.where({ id: 'ALT_DELETE' });
- const esc = data.key.where({ id: 'ESC' });
- return `Verify the Virtual Cursor is active by pressing ${altDelete.keystroke}. If it is not, exit Forms Mode to activate the Virtual Cursor by pressing ${esc.keystroke}.`;
- },
- },
- {
- at: 'jaws',
- mode: 'interaction',
- render: data => {
- const altDelete = data.key.where({ id: 'ALT_DELETE' });
- const insZ = data.key.where({ id: 'INS_Z' });
- return `Verify the PC Cursor is active by pressing ${altDelete.keystroke}. If it is not, turn off the Virtual Cursor by pressing ${insZ.keystroke}.`;
- },
- },
- {
- at: 'nvda',
- mode: 'reading',
- render: data => {
- const esc = data.key.where({ id: 'ESC' });
- return `Ensure NVDA is in browse mode by pressing ${esc.keystroke}. Note: This command has no effect if NVDA is already in browse mode.`;
- },
- },
- {
- at: 'nvda',
- mode: 'interaction',
- render: data => {
- const insSpace = data.key.where({ id: 'INS_SPACE' });
- return `If NVDA did not make the focus mode sound when the test page loaded, press ${insSpace.keystroke} to turn focus mode on.`;
- },
- },
- {
- at: 'voiceover_macos',
- mode: 'reading',
- render: data => {
- const left = data.key.where({ id: 'LEFT' });
- const right = data.key.where({ id: 'RIGHT' });
- return `Toggle Quick Nav ON by pressing the ${left.keystroke} and ${right.keystroke} keys at the same time.`;
- },
- },
- {
- at: 'voiceover_macos',
- mode: 'interaction',
- render: data => {
- const left = data.key.where({ id: 'LEFT' });
- const right = data.key.where({ id: 'RIGHT' });
- return `Toggle Quick Nav OFF by pressing the ${left.keystroke} and ${right.keystroke} keys at the same time.`;
- },
- },
-]);
+const MODE_INSTRUCTION_TEMPLATES_QUERYABLE = support => {
+ const collection = [];
+ support.ats.forEach(at => {
+ const modes = Object.keys(at.settings);
+ modes.forEach(mode => {
+ collection.push({
+ at: at.key,
+ mode: mode,
+ render: [at.defaultConfigurationInstructionsHTML, ...at.settings[mode].instructions],
+ });
+ });
+
+ // Accounting for modeless AT configurations
+ collection.push({
+ at: at.key,
+ mode: 'defaultMode',
+ render: [at.defaultConfigurationInstructionsHTML],
+ });
+ });
+
+ return Queryable.from('modeInstructionTemplate', collection);
+};
/**
* @param {T} maybeDefined
@@ -1157,39 +1404,51 @@ function mapDefined(maybeDefined, goal) {
* @param {Queryable} data.script
* @param {object} data.support
* @param {Queryable<{key: string, name: string}>} data.support.at
- * @param {Queryable<{key: string, name: string}>} data.support.atGroup
+ * @param {Queryable<{key: string, name: string}>} data.support.atGroups
* @param {object} [options]
* @param {function(string): void} [options.addTestError]
* @returns {AriaATValidated.Test}
*/
function validateTest(testParsed, data, { addTestError = () => {} } = {}) {
- if (!data.command.where({ task: testParsed.task })) {
- addTestError(`"${testParsed.task}" does not exist in commands.csv file.`);
+ const assertionStatementTokenPattern = /\{([^{}]+)}/g;
+ const replacedAssertionStatement = (assertionStatement, matches, replacements) => {
+ matches.forEach(
+ match =>
+ (assertionStatement = assertionStatement.replaceAll(
+ match,
+ replacements[match.slice(1, -1)]
+ ))
+ );
+ return assertionStatement;
+ };
+
+ if (!data.command.where({ testId: testParsed.testId })) {
+ addTestError(`"${testParsed.testId}" does not exist in *-commands.csv file.`);
}
- testParsed.target.at.forEach(at => {
- if (!data.support.atGroup.where({ key: at.key })) {
+ testParsed.target.ats.forEach(at => {
+ if (!data.support.atGroups.where({ key: at.key })) {
addTestError(`"${at.key}" is not valid value for "appliesTo" property.`);
}
if (
!data.command.where({
- task: testParsed.task,
target: {
- at: { key: at.key },
- mode: testParsed.target.mode,
+ at: { key: at.key, settings: at.settings },
},
})
) {
addTestError(
- `command is missing for the combination of task: "${testParsed.task}", mode: "${testParsed.target.mode}", and AT: "${at.key}"`
+ `command is missing for the combination of task: "${testParsed.testId}", mode: "${at.settings}", and AT: "${at.key}"`
);
}
});
- if (!data.mode.where(testParsed.target.mode)) {
- addTestError(`"${testParsed.target.mode}" is not valid value for "mode" property.`);
- }
+ testParsed.target.ats.forEach(at => {
+ if (!data.mode.where(at.settings)) {
+ addTestError(`"${at.settings}" is not valid value for "settings" property.`);
+ }
+ });
const references = testParsed.references.filter(({ refId }) => {
if (!data.reference.where({ refId })) {
@@ -1199,25 +1458,55 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) {
return true;
});
- if (testParsed.setupScript && !data.script.where({ name: testParsed.setupScript.name })) {
+ if (testParsed.setupScript && !data.script.where({ name: testParsed.setupScript.script })) {
addTestError(
- `Setup script does not exist: "${testParsed.setupScript.name}" for task "${testParsed.task}"`
+ `Setup script does not exist: "${testParsed.setupScript.script}" for task "${testParsed.testId}"`
);
}
const assertions = testParsed.assertions.map(assertion => {
+ assertion.tokenizedAssertionStatements = {};
+
+ // There are assertion tokens to account for
+ if (assertion.assertionStatement.includes('|')) {
+ const [genericAssertionStatement, tokenizedAssertionStatement] =
+ assertion.assertionStatement.split('|');
+
+ // Set fallback just in case tokenized statement or properties do not exist
+ assertion.assertionStatement = genericAssertionStatement;
+
+ if (tokenizedAssertionStatement) {
+ const matches = tokenizedAssertionStatement.match(assertionStatementTokenPattern);
+
+ testParsed.target.ats.forEach(at => {
+ const { assertionTokens } = data.support.ats.where({ key: at.key });
+ const tokensExist =
+ assertionTokens && matches.every(match => assertionTokens[match.slice(1, -1)]);
+
+ if (tokensExist) {
+ assertion.tokenizedAssertionStatements[at.key] = replacedAssertionStatement(
+ tokenizedAssertionStatement,
+ matches,
+ assertionTokens
+ );
+ }
+ });
+ }
+ }
+
if (
typeof assertion.priority === 'string' ||
- (assertion.priority !== 1 && assertion.priority !== 2)
+ (assertion.priority !== 1 && assertion.priority !== 2 && assertion.priority !== 3)
) {
addTestError(
- `Level value must be 1 or 2, value found was "${assertion.priority}" for assertion "${assertion.expectation}" (NOTE: level 2 defined for this assertion).`
+ `Level value must be 1, 2 or 3, value found was "${assertion.priority}" for assertion "${assertion.assertionStatement}" (NOTE: Priority 3 defined for this assertion).`
);
return {
- priority: 2,
- expectation: assertion.expectation,
+ ...assertion,
+ priority: 3,
};
}
+
return assertion;
});
@@ -1231,19 +1520,18 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) {
...data.reference.where({ refId: ref.refId }),
})),
target: {
- at: testParsed.target.at.map(at => ({
+ ats: testParsed.target.ats.map(at => ({
...at,
- ...mapDefined(data.support.at.where({ key: at.key }), ({ name }) => ({
+ ...mapDefined(data.support.ats.where({ key: at.key }), ({ name }) => ({
name,
})),
})),
- mode: testParsed.target.mode,
},
setupScript:
- testParsed.setupScript && data.script.where({ name: testParsed.setupScript.name })
+ testParsed.setupScript && data.script.where({ name: testParsed.setupScript.script })
? {
...testParsed.setupScript,
- ...data.script.where({ name: testParsed.setupScript.name }),
+ ...data.script.where({ name: testParsed.setupScript.script }),
}
: undefined,
assertions,
@@ -1254,33 +1542,31 @@ function validateTest(testParsed, data, { addTestError = () => {} } = {}) {
* @param {object} data
* @param {AriaATValidated.Test} data.test
* @param {AriaATValidated.Command} data.command
- * @param {Queryable} data.key
+ * @param {Queryable<{screenText: string, instructions: [string]}>} data.supportAt
* @param {Queryable<{name: string, path: string}>} data.example
* @param {Queryable<{at: string, mode: string, render: function({key: *}): string}>} data.modeInstructionTemplate
* @returns {AriaATFile.CollectedTest}
*/
-function collectTestData({ test, command, key, example, modeInstructionTemplate }) {
+function collectTestData({ test, command, supportAt, example, modeInstructionTemplate }) {
return {
info: {
testId: test.testId,
- task: test.task,
title: test.title,
references: test.references,
+ presentationNumber: test.presentationNumber,
},
target: {
...test.target,
- at: command.target.at,
+ at: { ...command.target.at, raw: supportAt.where({ key: command.target.at.key }) },
referencePage: example.where({ name: test.setupScript ? test.setupScript.name : '' }).path,
setupScript: test.setupScript,
},
instructions: {
- ...test.instructions,
- mode: modeInstructionTemplate
- .where({
- at: command.target.at.key,
- mode: command.target.mode,
- })
- .render({ key }),
+ instructions: test.instructions,
+ mode: modeInstructionTemplate.where({
+ at: command.target.at.key,
+ mode: command.target.at.settings,
+ }).render,
},
commands: command.commands,
assertions: test.assertions,
@@ -1296,16 +1582,18 @@ function encodeText(text) {
/**
* @param {AriaATFile.CollectedTest} test
+ * @param {string} testPlanBuildDirectory
+ * @param {string} fileName
+ * @returns {{path: string, content: Uint8Array}}
*/
-function createCollectedTestFile(test, testPlanBuildDirectory) {
+function createCollectedTestFile(test, testPlanBuildDirectory, fileName = null) {
+ const testPresentationNumber = parseInt(test.info.presentationNumber).toString().padStart(2, '0');
+ const testFileName =
+ fileName ||
+ `test-${testPresentationNumber}-${test.info.testId}-${test.target.at.key}-${test.target.at.settings}`;
+
return {
- path: path.join(
- testPlanBuildDirectory,
- `test-${test.info.testId.toString().padStart(2, '0')}-${test.info.task.replace(
- /\s+/g,
- '-'
- )}-${test.target.mode}-${test.target.at.key}.collected.json`
- ),
+ path: path.join(testPlanBuildDirectory, `${testFileName}.collected.json`),
content: encodeText(beautify(test, null, 2, 40)),
};
}
@@ -1313,23 +1601,18 @@ function createCollectedTestFile(test, testPlanBuildDirectory) {
/**
* @param {AriaATFile.CollectedTest} test
* @param {string} testPlanBuildDirectory
+ * @param {string} fileName
* @returns {{path: string, content: Uint8Array}}
*/
-function createCollectedTestHtmlFile(test, testPlanBuildDirectory) {
- const testJsonFileName = `test-${test.info.testId
- .toString()
- .padStart(2, '0')}-${test.info.task.replace(/\s+/g, '-')}-${test.target.mode}-${
- test.target.at.key
- }.collected.json`;
+function createCollectedTestHtmlFile(test, testPlanBuildDirectory, fileName = null) {
+ const testPresentationNumber = parseInt(test.info.presentationNumber).toString().padStart(2, '0');
+ const testFileName =
+ fileName ||
+ `test-${testPresentationNumber}-${test.info.testId}-${test.target.at.key}-${test.target.at.settings}`;
+
return {
- path: path.join(
- testPlanBuildDirectory,
- `test-${test.info.testId.toString().padStart(2, '0')}-${test.info.task.replace(
- /\s+/g,
- '-'
- )}-${test.target.mode}-${test.target.at.key}.collected.html`
- ),
- content: encodeText(renderCollectedTestHtml(test, testJsonFileName)),
+ path: path.join(testPlanBuildDirectory, `${testFileName}.collected.html`),
+ content: encodeText(renderCollectedTestHtml(test, `${testFileName}.collected.json`)),
};
}
diff --git a/package.json b/package.json
index 3f41e5e66..95d0dd012 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,9 @@
"count": "node ./scripts/count-assertions.js",
"lint": "eslint .",
"build": "npm run create-all-tests && npm run review-tests",
- "validate": "npm run create-all-tests -- --validate",
+ "build-v1": "npm run create-all-tests -- --version1 --verbose && npm run review-tests",
+ "build-v2": "npm run create-all-tests -- --version2 --verbose && npm run review-tests",
+ "validate": "npm run create-all-tests -- --verbose --validate",
"cleanup": "rimraf build",
"create-all-tests": "cross-var node scripts/create-all-tests.js --testplan=$npm_config_testplan",
"review-tests": "cross-var node --experimental-modules scripts/test-reviewer.mjs --testplan=$npm_config_testplan",
diff --git a/scripts/create-all-tests.js b/scripts/create-all-tests.js
index aa4d45305..c58266148 100644
--- a/scripts/create-all-tests.js
+++ b/scripts/create-all-tests.js
@@ -2,7 +2,12 @@ const path = require('path');
const fse = require('fs-extra');
-const { processTestDirectory } = require('../lib/data/process-test-directory');
+const {
+ processTestDirectory: processTestDirectoryV2,
+} = require('../lib/data/process-test-directory');
+const {
+ processTestDirectory: processTestDirectoryV1,
+} = require('../lib/data/process-test-directory-v1');
const args = require('minimist')(process.argv.slice(2), {
alias: {
@@ -10,6 +15,8 @@ const args = require('minimist')(process.argv.slice(2), {
t: 'testplan',
v: 'verbose',
V: 'validate',
+ v1: 'version1',
+ v2: 'version2',
},
});
@@ -26,6 +33,10 @@ if (args.help) {
Generate tests and view a detailed report summary.
-V, --validate
Determine whether current test plans are valid (no errors present).
+ -v1, --version1
+ Build the tests with the v1 format of the tests
+ -v2, --version2
+ Build the tests with the v2 format of the tests
`);
process.exit();
}
@@ -36,8 +47,11 @@ async function main() {
// on some OSes, it seems the `npm_config_testplan` environment variable will come back as the actual variable name rather than empty if it does not exist
const TARGET_TEST_PLAN =
args.testplan && !args.testplan.includes('npm_config_testplan') ? args.testplan : null; // individual test plan to generate test assets for
+
const VERBOSE_CHECK = !!args.verbose;
const VALIDATE_CHECK = !!args.validate;
+ const V1_CHECK = !!args.v1;
+ const V2_CHECK = !!args.v2;
const scriptsDirectory = path.dirname(__filename);
const rootDirectory = path.join(scriptsDirectory, '..');
@@ -58,15 +72,44 @@ async function main() {
}
const filteredTests = await Promise.all(
- filteredTestPlans.map(directory =>
- processTestDirectory({
- directory: path.join('tests', directory),
- args,
- }).catch(error => {
- error.directory = directory;
- throw error;
- })
- )
+ filteredTestPlans.map(directory => {
+ let FALLBACK_V1_CHECK = false;
+ let FALLBACK_V2_CHECK = false;
+
+ // Check if files exist for doing v2 build by default first, then try v1
+ if (!V1_CHECK && !V2_CHECK) {
+ // Use existence of assertions.csv to determine if v2 format files exist for now
+ const assertionsCsvPath = path.join(
+ __dirname,
+ '../',
+ 'tests',
+ directory,
+ 'data',
+ 'assertions.csv'
+ );
+
+ if (fse.existsSync(assertionsCsvPath)) FALLBACK_V2_CHECK = true;
+ else FALLBACK_V1_CHECK = true;
+ }
+
+ if (FALLBACK_V2_CHECK || V2_CHECK) {
+ return processTestDirectoryV2({
+ directory: path.join('tests', directory),
+ args,
+ }).catch(error => {
+ error.directory = directory;
+ throw error;
+ });
+ } else if (FALLBACK_V1_CHECK || V1_CHECK) {
+ return processTestDirectoryV1({
+ directory: path.join('tests', directory),
+ args,
+ }).catch(error => {
+ error.directory = directory;
+ throw error;
+ });
+ }
+ })
).catch(error => {
console.error(`ERROR: Unhandled exception thrown while processing "${error.directory}".`);
if (!VERBOSE_CHECK) {
diff --git a/scripts/review-index-template.mustache b/scripts/review-index-template.mustache
index ab76db06f..ef19e13db 100644
--- a/scripts/review-index-template.mustache
+++ b/scripts/review-index-template.mustache
@@ -38,6 +38,7 @@
+ Title
Pattern
Index Page
Review Page
@@ -46,9 +47,10 @@
{{#patterns}}
- {{name}}
- Index
- Review
+ {{title}}
+ {{patternName}}
+ Index
+ Review
{{numberOfTests}}
{{commitDescription}}
diff --git a/scripts/review-template.mustache b/scripts/review-template.mustache
index fdc248967..74063f23b 100644
--- a/scripts/review-template.mustache
+++ b/scripts/review-template.mustache
@@ -2,7 +2,7 @@
- Test plan review for pattern: {{pattern}}
+ {{title}} Test Plan | For Pattern: {{pattern}}